2024-11-20 08:59:14,675 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-20 08:59:14,690 main DEBUG Took 0.012908 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-20 08:59:14,691 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-20 08:59:14,691 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-20 08:59:14,692 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-20 08:59:14,694 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 08:59:14,703 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-20 08:59:14,718 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 08:59:14,720 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 08:59:14,721 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 08:59:14,721 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 08:59:14,722 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 08:59:14,723 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 08:59:14,724 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 08:59:14,724 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 08:59:14,725 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 08:59:14,725 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 08:59:14,727 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 08:59:14,727 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 08:59:14,728 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 08:59:14,728 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 08:59:14,729 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 08:59:14,730 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 08:59:14,730 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 08:59:14,731 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 08:59:14,731 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 08:59:14,732 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 08:59:14,733 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 08:59:14,733 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 08:59:14,734 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 08:59:14,734 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 08:59:14,735 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 08:59:14,735 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-20 08:59:14,737 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 08:59:14,739 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-20 08:59:14,742 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-20 08:59:14,742 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-20 08:59:14,744 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-20 08:59:14,744 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-20 08:59:14,756 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-20 08:59:14,758 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-20 08:59:14,760 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-20 08:59:14,761 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-20 08:59:14,761 main DEBUG createAppenders(={Console}) 2024-11-20 08:59:14,762 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad initialized 2024-11-20 08:59:14,762 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-20 08:59:14,762 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad OK. 2024-11-20 08:59:14,763 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-20 08:59:14,763 main DEBUG OutputStream closed 2024-11-20 08:59:14,763 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-20 08:59:14,764 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-20 08:59:14,764 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@55634720 OK 2024-11-20 08:59:14,847 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-20 08:59:14,849 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-20 08:59:14,850 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-20 08:59:14,851 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-20 08:59:14,852 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-20 08:59:14,852 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-20 08:59:14,852 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-20 08:59:14,853 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-20 08:59:14,853 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-20 08:59:14,853 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-20 08:59:14,854 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-20 08:59:14,854 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-20 08:59:14,854 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-20 08:59:14,855 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-20 08:59:14,855 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-20 08:59:14,856 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-20 08:59:14,856 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-20 08:59:14,857 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-20 08:59:14,859 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-20 08:59:14,859 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@19835e64) with optional ClassLoader: null 2024-11-20 08:59:14,860 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-20 08:59:14,861 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@19835e64] started OK. 2024-11-20T08:59:15,156 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d52d975-8094-4c70-a6d2-7d2f656df145 2024-11-20 08:59:15,160 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-20 08:59:15,160 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-20T08:59:15,172 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.TestAcidGuaranteesWithNoInMemCompaction timeout: 13 mins 2024-11-20T08:59:15,197 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-20T08:59:15,219 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d52d975-8094-4c70-a6d2-7d2f656df145/cluster_0d88e112-ef41-ee03-4061-1e732edd46d6, deleteOnExit=true 2024-11-20T08:59:15,219 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-20T08:59:15,220 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d52d975-8094-4c70-a6d2-7d2f656df145/test.cache.data in system properties and HBase conf 2024-11-20T08:59:15,221 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d52d975-8094-4c70-a6d2-7d2f656df145/hadoop.tmp.dir in system properties and HBase conf 2024-11-20T08:59:15,222 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d52d975-8094-4c70-a6d2-7d2f656df145/hadoop.log.dir in system properties and HBase conf 2024-11-20T08:59:15,222 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d52d975-8094-4c70-a6d2-7d2f656df145/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-20T08:59:15,223 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d52d975-8094-4c70-a6d2-7d2f656df145/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-20T08:59:15,224 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-20T08:59:15,324 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-20T08:59:15,422 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-20T08:59:15,426 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d52d975-8094-4c70-a6d2-7d2f656df145/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-20T08:59:15,427 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d52d975-8094-4c70-a6d2-7d2f656df145/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-20T08:59:15,427 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d52d975-8094-4c70-a6d2-7d2f656df145/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-20T08:59:15,428 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d52d975-8094-4c70-a6d2-7d2f656df145/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-20T08:59:15,428 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d52d975-8094-4c70-a6d2-7d2f656df145/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-20T08:59:15,429 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d52d975-8094-4c70-a6d2-7d2f656df145/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-20T08:59:15,429 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d52d975-8094-4c70-a6d2-7d2f656df145/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-20T08:59:15,430 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d52d975-8094-4c70-a6d2-7d2f656df145/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-20T08:59:15,430 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d52d975-8094-4c70-a6d2-7d2f656df145/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-20T08:59:15,430 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d52d975-8094-4c70-a6d2-7d2f656df145/nfs.dump.dir in system properties and HBase conf 2024-11-20T08:59:15,431 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d52d975-8094-4c70-a6d2-7d2f656df145/java.io.tmpdir in system properties and HBase conf 2024-11-20T08:59:15,431 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d52d975-8094-4c70-a6d2-7d2f656df145/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-20T08:59:15,432 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d52d975-8094-4c70-a6d2-7d2f656df145/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-20T08:59:15,432 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d52d975-8094-4c70-a6d2-7d2f656df145/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-20T08:59:16,332 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-20T08:59:16,426 INFO [Time-limited test {}] log.Log(170): Logging initialized @2568ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-20T08:59:16,512 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-20T08:59:16,599 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-20T08:59:16,623 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-20T08:59:16,624 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-20T08:59:16,626 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-20T08:59:16,640 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-20T08:59:16,643 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d52d975-8094-4c70-a6d2-7d2f656df145/hadoop.log.dir/,AVAILABLE} 2024-11-20T08:59:16,644 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-20T08:59:16,864 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4ded99d3{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d52d975-8094-4c70-a6d2-7d2f656df145/java.io.tmpdir/jetty-localhost-46741-hadoop-hdfs-3_4_1-tests_jar-_-any-7870295060399593877/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-20T08:59:16,872 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:46741} 2024-11-20T08:59:16,873 INFO [Time-limited test {}] server.Server(415): Started @3015ms 2024-11-20T08:59:17,337 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-20T08:59:17,346 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-20T08:59:17,347 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-20T08:59:17,347 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-20T08:59:17,348 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-20T08:59:17,348 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4fd7fa7c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d52d975-8094-4c70-a6d2-7d2f656df145/hadoop.log.dir/,AVAILABLE} 2024-11-20T08:59:17,349 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@435f9f96{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-20T08:59:17,489 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4269bd31{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d52d975-8094-4c70-a6d2-7d2f656df145/java.io.tmpdir/jetty-localhost-42017-hadoop-hdfs-3_4_1-tests_jar-_-any-6344794271242224097/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-20T08:59:17,490 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@353d64f2{HTTP/1.1, (http/1.1)}{localhost:42017} 2024-11-20T08:59:17,490 INFO [Time-limited test {}] server.Server(415): Started @3633ms 2024-11-20T08:59:17,550 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-20T08:59:18,078 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d52d975-8094-4c70-a6d2-7d2f656df145/cluster_0d88e112-ef41-ee03-4061-1e732edd46d6/data/data1/current/BP-1443621306-172.17.0.2-1732093156059/current, will proceed with Du for space computation calculation, 2024-11-20T08:59:18,078 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d52d975-8094-4c70-a6d2-7d2f656df145/cluster_0d88e112-ef41-ee03-4061-1e732edd46d6/data/data2/current/BP-1443621306-172.17.0.2-1732093156059/current, will proceed with Du for space computation calculation, 2024-11-20T08:59:18,122 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-20T08:59:18,176 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xc23da7bf9900c97 with lease ID 0x9b9c748656941e9e: Processing first storage report for DS-e81a7599-9758-427f-a85e-bc7af53a6144 from datanode DatanodeRegistration(127.0.0.1:40033, datanodeUuid=4ae02408-43a8-4abb-8a76-2d8c0890ff10, infoPort=33263, infoSecurePort=0, ipcPort=35795, storageInfo=lv=-57;cid=testClusterID;nsid=192430032;c=1732093156059) 2024-11-20T08:59:18,178 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xc23da7bf9900c97 with lease ID 0x9b9c748656941e9e: from storage DS-e81a7599-9758-427f-a85e-bc7af53a6144 node DatanodeRegistration(127.0.0.1:40033, datanodeUuid=4ae02408-43a8-4abb-8a76-2d8c0890ff10, infoPort=33263, infoSecurePort=0, ipcPort=35795, storageInfo=lv=-57;cid=testClusterID;nsid=192430032;c=1732093156059), blocks: 0, hasStaleStorage: true, processing time: 3 msecs, invalidatedBlocks: 0 2024-11-20T08:59:18,179 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xc23da7bf9900c97 with lease ID 0x9b9c748656941e9e: Processing first storage report for DS-bece994a-8124-43a0-8817-3b2a0d5fc946 from datanode DatanodeRegistration(127.0.0.1:40033, datanodeUuid=4ae02408-43a8-4abb-8a76-2d8c0890ff10, infoPort=33263, infoSecurePort=0, ipcPort=35795, storageInfo=lv=-57;cid=testClusterID;nsid=192430032;c=1732093156059) 2024-11-20T08:59:18,179 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xc23da7bf9900c97 with lease ID 0x9b9c748656941e9e: from storage DS-bece994a-8124-43a0-8817-3b2a0d5fc946 node DatanodeRegistration(127.0.0.1:40033, datanodeUuid=4ae02408-43a8-4abb-8a76-2d8c0890ff10, infoPort=33263, infoSecurePort=0, ipcPort=35795, storageInfo=lv=-57;cid=testClusterID;nsid=192430032;c=1732093156059), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-20T08:59:18,196 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d52d975-8094-4c70-a6d2-7d2f656df145 2024-11-20T08:59:18,283 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d52d975-8094-4c70-a6d2-7d2f656df145/cluster_0d88e112-ef41-ee03-4061-1e732edd46d6/zookeeper_0, clientPort=52155, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d52d975-8094-4c70-a6d2-7d2f656df145/cluster_0d88e112-ef41-ee03-4061-1e732edd46d6/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d52d975-8094-4c70-a6d2-7d2f656df145/cluster_0d88e112-ef41-ee03-4061-1e732edd46d6/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-20T08:59:18,295 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=52155 2024-11-20T08:59:18,311 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-20T08:59:18,315 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-20T08:59:18,578 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073741825_1001 (size=7) 2024-11-20T08:59:18,988 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a with version=8 2024-11-20T08:59:18,989 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/hbase-staging 2024-11-20T08:59:19,093 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-20T08:59:19,377 INFO [Time-limited test {}] client.ConnectionUtils(128): master/be1eb2620e0e:0 server-side Connection retries=45 2024-11-20T08:59:19,390 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-20T08:59:19,391 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-20T08:59:19,396 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-20T08:59:19,396 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-20T08:59:19,396 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-20T08:59:19,569 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-20T08:59:19,650 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-20T08:59:19,660 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-20T08:59:19,665 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-20T08:59:19,695 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 11012 (auto-detected) 2024-11-20T08:59:19,696 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-20T08:59:19,717 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:44373 2024-11-20T08:59:19,741 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:44373 connecting to ZooKeeper ensemble=127.0.0.1:52155 2024-11-20T08:59:19,781 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:443730x0, quorum=127.0.0.1:52155, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-20T08:59:19,784 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:44373-0x1001427dc570000 connected 2024-11-20T08:59:19,816 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-20T08:59:19,819 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-20T08:59:19,830 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:44373-0x1001427dc570000, quorum=127.0.0.1:52155, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-20T08:59:19,834 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a, hbase.cluster.distributed=false 2024-11-20T08:59:19,860 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:44373-0x1001427dc570000, quorum=127.0.0.1:52155, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-20T08:59:19,868 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=44373 2024-11-20T08:59:19,871 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=44373 2024-11-20T08:59:19,872 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=44373 2024-11-20T08:59:19,874 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=44373 2024-11-20T08:59:19,875 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=44373 2024-11-20T08:59:20,005 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/be1eb2620e0e:0 server-side Connection retries=45 2024-11-20T08:59:20,007 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-20T08:59:20,007 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-20T08:59:20,007 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-20T08:59:20,008 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-20T08:59:20,008 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-20T08:59:20,011 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-20T08:59:20,013 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-20T08:59:20,014 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:34223 2024-11-20T08:59:20,016 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:34223 connecting to ZooKeeper ensemble=127.0.0.1:52155 2024-11-20T08:59:20,018 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-20T08:59:20,021 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-20T08:59:20,027 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:342230x0, quorum=127.0.0.1:52155, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-20T08:59:20,029 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:34223-0x1001427dc570001 connected 2024-11-20T08:59:20,029 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34223-0x1001427dc570001, quorum=127.0.0.1:52155, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-20T08:59:20,036 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-20T08:59:20,047 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-20T08:59:20,051 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34223-0x1001427dc570001, quorum=127.0.0.1:52155, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-20T08:59:20,057 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34223-0x1001427dc570001, quorum=127.0.0.1:52155, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-20T08:59:20,058 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=34223 2024-11-20T08:59:20,059 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=34223 2024-11-20T08:59:20,060 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=34223 2024-11-20T08:59:20,061 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=34223 2024-11-20T08:59:20,061 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=34223 2024-11-20T08:59:20,080 DEBUG [M:0;be1eb2620e0e:44373 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;be1eb2620e0e:44373 2024-11-20T08:59:20,082 INFO [master/be1eb2620e0e:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/be1eb2620e0e,44373,1732093159150 2024-11-20T08:59:20,089 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44373-0x1001427dc570000, quorum=127.0.0.1:52155, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-20T08:59:20,089 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34223-0x1001427dc570001, quorum=127.0.0.1:52155, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-20T08:59:20,091 DEBUG [master/be1eb2620e0e:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:44373-0x1001427dc570000, quorum=127.0.0.1:52155, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/be1eb2620e0e,44373,1732093159150 2024-11-20T08:59:20,114 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34223-0x1001427dc570001, quorum=127.0.0.1:52155, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-20T08:59:20,114 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44373-0x1001427dc570000, quorum=127.0.0.1:52155, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T08:59:20,114 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34223-0x1001427dc570001, quorum=127.0.0.1:52155, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T08:59:20,115 DEBUG [master/be1eb2620e0e:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:44373-0x1001427dc570000, quorum=127.0.0.1:52155, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-20T08:59:20,117 INFO [master/be1eb2620e0e:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/be1eb2620e0e,44373,1732093159150 from backup master directory 2024-11-20T08:59:20,120 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44373-0x1001427dc570000, quorum=127.0.0.1:52155, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/be1eb2620e0e,44373,1732093159150 2024-11-20T08:59:20,120 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34223-0x1001427dc570001, quorum=127.0.0.1:52155, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-20T08:59:20,120 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44373-0x1001427dc570000, quorum=127.0.0.1:52155, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-20T08:59:20,120 WARN [master/be1eb2620e0e:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-20T08:59:20,121 INFO [master/be1eb2620e0e:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=be1eb2620e0e,44373,1732093159150 2024-11-20T08:59:20,123 INFO [master/be1eb2620e0e:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-20T08:59:20,125 INFO [master/be1eb2620e0e:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-20T08:59:20,191 DEBUG [master/be1eb2620e0e:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/hbase.id] with ID: caf50035-b19e-4cfd-90ea-dca559866882 2024-11-20T08:59:20,191 DEBUG [master/be1eb2620e0e:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/.tmp/hbase.id 2024-11-20T08:59:20,203 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073741826_1002 (size=42) 2024-11-20T08:59:20,605 DEBUG [master/be1eb2620e0e:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/.tmp/hbase.id]:[hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/hbase.id] 2024-11-20T08:59:20,656 INFO [master/be1eb2620e0e:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-20T08:59:20,661 INFO [master/be1eb2620e0e:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-20T08:59:20,683 INFO [master/be1eb2620e0e:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 19ms. 2024-11-20T08:59:20,687 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44373-0x1001427dc570000, quorum=127.0.0.1:52155, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T08:59:20,687 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34223-0x1001427dc570001, quorum=127.0.0.1:52155, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T08:59:20,705 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073741827_1003 (size=196) 2024-11-20T08:59:20,726 INFO [master/be1eb2620e0e:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-20T08:59:20,729 INFO [master/be1eb2620e0e:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-20T08:59:20,747 DEBUG [master/be1eb2620e0e:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T08:59:20,753 INFO [master/be1eb2620e0e:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T08:59:20,791 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073741828_1004 (size=1189) 2024-11-20T08:59:21,213 INFO [master/be1eb2620e0e:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/MasterData/data/master/store 2024-11-20T08:59:21,235 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073741829_1005 (size=34) 2024-11-20T08:59:21,641 INFO [master/be1eb2620e0e:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-20T08:59:21,645 DEBUG [master/be1eb2620e0e:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T08:59:21,646 DEBUG [master/be1eb2620e0e:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-20T08:59:21,646 INFO [master/be1eb2620e0e:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T08:59:21,647 DEBUG [master/be1eb2620e0e:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T08:59:21,648 DEBUG [master/be1eb2620e0e:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-20T08:59:21,648 DEBUG [master/be1eb2620e0e:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T08:59:21,649 INFO [master/be1eb2620e0e:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T08:59:21,650 DEBUG [master/be1eb2620e0e:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732093161646Disabling compacts and flushes for region at 1732093161646Disabling writes for close at 1732093161648 (+2 ms)Writing region close event to WAL at 1732093161649 (+1 ms)Closed at 1732093161649 2024-11-20T08:59:21,652 WARN [master/be1eb2620e0e:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/MasterData/data/master/store/.initializing 2024-11-20T08:59:21,653 DEBUG [master/be1eb2620e0e:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/MasterData/WALs/be1eb2620e0e,44373,1732093159150 2024-11-20T08:59:21,662 INFO [master/be1eb2620e0e:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-20T08:59:21,679 INFO [master/be1eb2620e0e:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=be1eb2620e0e%2C44373%2C1732093159150, suffix=, logDir=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/MasterData/WALs/be1eb2620e0e,44373,1732093159150, archiveDir=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/MasterData/oldWALs, maxLogs=10 2024-11-20T08:59:21,705 DEBUG [master/be1eb2620e0e:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/MasterData/WALs/be1eb2620e0e,44373,1732093159150/be1eb2620e0e%2C44373%2C1732093159150.1732093161684, exclude list is [], retry=0 2024-11-20T08:59:21,724 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40033,DS-e81a7599-9758-427f-a85e-bc7af53a6144,DISK] 2024-11-20T08:59:21,727 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-20T08:59:21,770 INFO [master/be1eb2620e0e:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/MasterData/WALs/be1eb2620e0e,44373,1732093159150/be1eb2620e0e%2C44373%2C1732093159150.1732093161684 2024-11-20T08:59:21,771 DEBUG [master/be1eb2620e0e:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33263:33263)] 2024-11-20T08:59:21,772 DEBUG [master/be1eb2620e0e:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-20T08:59:21,773 DEBUG [master/be1eb2620e0e:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T08:59:21,777 DEBUG [master/be1eb2620e0e:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-20T08:59:21,778 DEBUG [master/be1eb2620e0e:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-20T08:59:21,821 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-20T08:59:21,852 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-20T08:59:21,857 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T08:59:21,860 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T08:59:21,861 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-20T08:59:21,865 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-20T08:59:21,865 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T08:59:21,866 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T08:59:21,867 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-20T08:59:21,870 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-20T08:59:21,870 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T08:59:21,871 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T08:59:21,872 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-20T08:59:21,875 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-20T08:59:21,876 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T08:59:21,877 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T08:59:21,877 DEBUG [master/be1eb2620e0e:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-20T08:59:21,881 DEBUG [master/be1eb2620e0e:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-20T08:59:21,883 DEBUG [master/be1eb2620e0e:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-20T08:59:21,889 DEBUG [master/be1eb2620e0e:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-20T08:59:21,890 DEBUG [master/be1eb2620e0e:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-20T08:59:21,893 DEBUG [master/be1eb2620e0e:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-20T08:59:21,898 DEBUG [master/be1eb2620e0e:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-20T08:59:21,903 DEBUG [master/be1eb2620e0e:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T08:59:21,904 INFO [master/be1eb2620e0e:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62108491, jitterRate=-0.07451136410236359}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-20T08:59:21,910 DEBUG [master/be1eb2620e0e:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732093161793Initializing all the Stores at 1732093161796 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732093161797 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732093161797Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732093161798 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732093161798Cleaning up temporary data from old regions at 1732093161890 (+92 ms)Region opened successfully at 1732093161910 (+20 ms) 2024-11-20T08:59:21,911 INFO [master/be1eb2620e0e:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-20T08:59:21,950 DEBUG [master/be1eb2620e0e:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@54b8555, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=be1eb2620e0e/172.17.0.2:0 2024-11-20T08:59:21,988 INFO [master/be1eb2620e0e:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-20T08:59:22,002 INFO [master/be1eb2620e0e:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-20T08:59:22,003 INFO [master/be1eb2620e0e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-20T08:59:22,006 INFO [master/be1eb2620e0e:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-20T08:59:22,008 INFO [master/be1eb2620e0e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-20T08:59:22,016 INFO [master/be1eb2620e0e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 7 msec 2024-11-20T08:59:22,016 INFO [master/be1eb2620e0e:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-20T08:59:22,046 INFO [master/be1eb2620e0e:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-20T08:59:22,057 DEBUG [master/be1eb2620e0e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44373-0x1001427dc570000, quorum=127.0.0.1:52155, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-20T08:59:22,060 DEBUG [master/be1eb2620e0e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-20T08:59:22,062 INFO [master/be1eb2620e0e:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-20T08:59:22,064 DEBUG [master/be1eb2620e0e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44373-0x1001427dc570000, quorum=127.0.0.1:52155, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-20T08:59:22,066 DEBUG [master/be1eb2620e0e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-20T08:59:22,069 INFO [master/be1eb2620e0e:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-20T08:59:22,073 DEBUG [master/be1eb2620e0e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44373-0x1001427dc570000, quorum=127.0.0.1:52155, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-20T08:59:22,076 DEBUG [master/be1eb2620e0e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-20T08:59:22,077 DEBUG [master/be1eb2620e0e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44373-0x1001427dc570000, quorum=127.0.0.1:52155, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-20T08:59:22,079 DEBUG [master/be1eb2620e0e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-20T08:59:22,100 DEBUG [master/be1eb2620e0e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44373-0x1001427dc570000, quorum=127.0.0.1:52155, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-20T08:59:22,102 DEBUG [master/be1eb2620e0e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-20T08:59:22,106 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44373-0x1001427dc570000, quorum=127.0.0.1:52155, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-20T08:59:22,106 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34223-0x1001427dc570001, quorum=127.0.0.1:52155, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-20T08:59:22,107 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44373-0x1001427dc570000, quorum=127.0.0.1:52155, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T08:59:22,107 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34223-0x1001427dc570001, quorum=127.0.0.1:52155, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T08:59:22,110 INFO [master/be1eb2620e0e:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=be1eb2620e0e,44373,1732093159150, sessionid=0x1001427dc570000, setting cluster-up flag (Was=false) 2024-11-20T08:59:22,124 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34223-0x1001427dc570001, quorum=127.0.0.1:52155, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T08:59:22,124 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44373-0x1001427dc570000, quorum=127.0.0.1:52155, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T08:59:22,132 DEBUG [master/be1eb2620e0e:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-20T08:59:22,137 DEBUG [master/be1eb2620e0e:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=be1eb2620e0e,44373,1732093159150 2024-11-20T08:59:22,144 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44373-0x1001427dc570000, quorum=127.0.0.1:52155, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T08:59:22,144 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34223-0x1001427dc570001, quorum=127.0.0.1:52155, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T08:59:22,151 DEBUG [master/be1eb2620e0e:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-20T08:59:22,155 DEBUG [master/be1eb2620e0e:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=be1eb2620e0e,44373,1732093159150 2024-11-20T08:59:22,164 INFO [master/be1eb2620e0e:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-20T08:59:22,170 INFO [RS:0;be1eb2620e0e:34223 {}] regionserver.HRegionServer(746): ClusterId : caf50035-b19e-4cfd-90ea-dca559866882 2024-11-20T08:59:22,174 DEBUG [RS:0;be1eb2620e0e:34223 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-20T08:59:22,180 DEBUG [RS:0;be1eb2620e0e:34223 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-20T08:59:22,180 DEBUG [RS:0;be1eb2620e0e:34223 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-20T08:59:22,183 DEBUG [RS:0;be1eb2620e0e:34223 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-20T08:59:22,184 DEBUG [RS:0;be1eb2620e0e:34223 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e923514, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=be1eb2620e0e/172.17.0.2:0 2024-11-20T08:59:22,199 DEBUG [RS:0;be1eb2620e0e:34223 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;be1eb2620e0e:34223 2024-11-20T08:59:22,203 INFO [RS:0;be1eb2620e0e:34223 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-20T08:59:22,203 INFO [RS:0;be1eb2620e0e:34223 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-20T08:59:22,203 DEBUG [RS:0;be1eb2620e0e:34223 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-20T08:59:22,206 INFO [RS:0;be1eb2620e0e:34223 {}] regionserver.HRegionServer(2659): reportForDuty to master=be1eb2620e0e,44373,1732093159150 with port=34223, startcode=1732093159960 2024-11-20T08:59:22,219 DEBUG [RS:0;be1eb2620e0e:34223 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-20T08:59:22,265 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44821, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-20T08:59:22,267 DEBUG [master/be1eb2620e0e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-20T08:59:22,272 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44373 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:22,280 INFO [master/be1eb2620e0e:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-20T08:59:22,288 INFO [master/be1eb2620e0e:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-20T08:59:22,296 DEBUG [master/be1eb2620e0e:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: be1eb2620e0e,44373,1732093159150 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-20T08:59:22,303 DEBUG [RS:0;be1eb2620e0e:34223 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-20T08:59:22,304 WARN [RS:0;be1eb2620e0e:34223 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-20T08:59:22,306 DEBUG [master/be1eb2620e0e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/be1eb2620e0e:0, corePoolSize=5, maxPoolSize=5 2024-11-20T08:59:22,306 DEBUG [master/be1eb2620e0e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/be1eb2620e0e:0, corePoolSize=5, maxPoolSize=5 2024-11-20T08:59:22,307 DEBUG [master/be1eb2620e0e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/be1eb2620e0e:0, corePoolSize=5, maxPoolSize=5 2024-11-20T08:59:22,307 DEBUG [master/be1eb2620e0e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/be1eb2620e0e:0, corePoolSize=5, maxPoolSize=5 2024-11-20T08:59:22,307 DEBUG [master/be1eb2620e0e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/be1eb2620e0e:0, corePoolSize=10, maxPoolSize=10 2024-11-20T08:59:22,307 DEBUG [master/be1eb2620e0e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/be1eb2620e0e:0, corePoolSize=1, maxPoolSize=1 2024-11-20T08:59:22,307 DEBUG [master/be1eb2620e0e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/be1eb2620e0e:0, corePoolSize=2, maxPoolSize=2 2024-11-20T08:59:22,307 DEBUG [master/be1eb2620e0e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/be1eb2620e0e:0, corePoolSize=1, maxPoolSize=1 2024-11-20T08:59:22,318 INFO [master/be1eb2620e0e:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732093192318 2024-11-20T08:59:22,319 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-20T08:59:22,320 INFO [master/be1eb2620e0e:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-20T08:59:22,320 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-20T08:59:22,321 INFO [master/be1eb2620e0e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-20T08:59:22,326 INFO [master/be1eb2620e0e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-20T08:59:22,326 INFO [master/be1eb2620e0e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-20T08:59:22,327 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T08:59:22,327 INFO [master/be1eb2620e0e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-20T08:59:22,327 INFO [master/be1eb2620e0e:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-20T08:59:22,327 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-20T08:59:22,328 INFO [master/be1eb2620e0e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-20T08:59:22,331 INFO [master/be1eb2620e0e:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-20T08:59:22,333 INFO [master/be1eb2620e0e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-20T08:59:22,333 INFO [master/be1eb2620e0e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-20T08:59:22,336 INFO [master/be1eb2620e0e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-20T08:59:22,336 INFO [master/be1eb2620e0e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-20T08:59:22,340 DEBUG [master/be1eb2620e0e:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/be1eb2620e0e:0:becomeActiveMaster-HFileCleaner.large.0-1732093162338,5,FailOnTimeoutGroup] 2024-11-20T08:59:22,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073741831_1007 (size=1321) 2024-11-20T08:59:22,345 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-20T08:59:22,346 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a 2024-11-20T08:59:22,348 DEBUG [master/be1eb2620e0e:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/be1eb2620e0e:0:becomeActiveMaster-HFileCleaner.small.0-1732093162340,5,FailOnTimeoutGroup] 2024-11-20T08:59:22,348 INFO [master/be1eb2620e0e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-20T08:59:22,348 INFO [master/be1eb2620e0e:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-20T08:59:22,349 INFO [master/be1eb2620e0e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-20T08:59:22,350 INFO [master/be1eb2620e0e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-20T08:59:22,358 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073741832_1008 (size=32) 2024-11-20T08:59:22,360 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T08:59:22,363 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-20T08:59:22,366 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-20T08:59:22,366 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T08:59:22,367 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T08:59:22,367 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-20T08:59:22,370 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-20T08:59:22,370 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T08:59:22,371 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T08:59:22,372 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-20T08:59:22,374 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-20T08:59:22,375 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T08:59:22,376 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T08:59:22,376 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-20T08:59:22,379 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-20T08:59:22,379 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T08:59:22,380 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T08:59:22,380 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-20T08:59:22,382 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/hbase/meta/1588230740 2024-11-20T08:59:22,382 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/hbase/meta/1588230740 2024-11-20T08:59:22,387 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-20T08:59:22,387 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-20T08:59:22,388 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-20T08:59:22,390 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-20T08:59:22,397 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T08:59:22,398 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61275548, jitterRate=-0.08692318201065063}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-20T08:59:22,401 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732093162360Initializing all the Stores at 1732093162362 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732093162362Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732093162362Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732093162362Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732093162363 (+1 ms)Cleaning up temporary data from old regions at 1732093162387 (+24 ms)Region opened successfully at 1732093162401 (+14 ms) 2024-11-20T08:59:22,401 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-20T08:59:22,401 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-20T08:59:22,401 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-20T08:59:22,402 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-20T08:59:22,402 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-20T08:59:22,405 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-20T08:59:22,405 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732093162401Disabling compacts and flushes for region at 1732093162401Disabling writes for close at 1732093162402 (+1 ms)Writing region close event to WAL at 1732093162404 (+2 ms)Closed at 1732093162405 (+1 ms) 2024-11-20T08:59:22,405 INFO [RS:0;be1eb2620e0e:34223 {}] regionserver.HRegionServer(2659): reportForDuty to master=be1eb2620e0e,44373,1732093159150 with port=34223, startcode=1732093159960 2024-11-20T08:59:22,408 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44373 {}] master.ServerManager(363): Checking decommissioned status of RegionServer be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:22,410 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-20T08:59:22,410 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-20T08:59:22,411 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44373 {}] master.ServerManager(517): Registering regionserver=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:22,418 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-20T08:59:22,421 DEBUG [RS:0;be1eb2620e0e:34223 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a 2024-11-20T08:59:22,421 DEBUG [RS:0;be1eb2620e0e:34223 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:40923 2024-11-20T08:59:22,421 DEBUG [RS:0;be1eb2620e0e:34223 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-20T08:59:22,427 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44373-0x1001427dc570000, quorum=127.0.0.1:52155, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-20T08:59:22,427 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-20T08:59:22,428 DEBUG [RS:0;be1eb2620e0e:34223 {}] zookeeper.ZKUtil(111): regionserver:34223-0x1001427dc570001, quorum=127.0.0.1:52155, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:22,428 WARN [RS:0;be1eb2620e0e:34223 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-20T08:59:22,428 INFO [RS:0;be1eb2620e0e:34223 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T08:59:22,429 DEBUG [RS:0;be1eb2620e0e:34223 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/WALs/be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:22,430 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [be1eb2620e0e,34223,1732093159960] 2024-11-20T08:59:22,431 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-20T08:59:22,459 INFO [RS:0;be1eb2620e0e:34223 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-20T08:59:22,474 INFO [RS:0;be1eb2620e0e:34223 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-20T08:59:22,481 INFO [RS:0;be1eb2620e0e:34223 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-20T08:59:22,481 INFO [RS:0;be1eb2620e0e:34223 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T08:59:22,482 INFO [RS:0;be1eb2620e0e:34223 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-20T08:59:22,488 INFO [RS:0;be1eb2620e0e:34223 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-20T08:59:22,490 INFO [RS:0;be1eb2620e0e:34223 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-20T08:59:22,490 DEBUG [RS:0;be1eb2620e0e:34223 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/be1eb2620e0e:0, corePoolSize=1, maxPoolSize=1 2024-11-20T08:59:22,490 DEBUG [RS:0;be1eb2620e0e:34223 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/be1eb2620e0e:0, corePoolSize=1, maxPoolSize=1 2024-11-20T08:59:22,491 DEBUG [RS:0;be1eb2620e0e:34223 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/be1eb2620e0e:0, corePoolSize=1, maxPoolSize=1 2024-11-20T08:59:22,491 DEBUG [RS:0;be1eb2620e0e:34223 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/be1eb2620e0e:0, corePoolSize=1, maxPoolSize=1 2024-11-20T08:59:22,491 DEBUG [RS:0;be1eb2620e0e:34223 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/be1eb2620e0e:0, corePoolSize=1, maxPoolSize=1 2024-11-20T08:59:22,491 DEBUG [RS:0;be1eb2620e0e:34223 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/be1eb2620e0e:0, corePoolSize=2, maxPoolSize=2 2024-11-20T08:59:22,491 DEBUG [RS:0;be1eb2620e0e:34223 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/be1eb2620e0e:0, corePoolSize=1, maxPoolSize=1 2024-11-20T08:59:22,491 DEBUG [RS:0;be1eb2620e0e:34223 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/be1eb2620e0e:0, corePoolSize=1, maxPoolSize=1 2024-11-20T08:59:22,492 DEBUG [RS:0;be1eb2620e0e:34223 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/be1eb2620e0e:0, corePoolSize=1, maxPoolSize=1 2024-11-20T08:59:22,492 DEBUG [RS:0;be1eb2620e0e:34223 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/be1eb2620e0e:0, corePoolSize=1, maxPoolSize=1 2024-11-20T08:59:22,492 DEBUG [RS:0;be1eb2620e0e:34223 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/be1eb2620e0e:0, corePoolSize=1, maxPoolSize=1 2024-11-20T08:59:22,492 DEBUG [RS:0;be1eb2620e0e:34223 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/be1eb2620e0e:0, corePoolSize=1, maxPoolSize=1 2024-11-20T08:59:22,492 DEBUG [RS:0;be1eb2620e0e:34223 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/be1eb2620e0e:0, corePoolSize=3, maxPoolSize=3 2024-11-20T08:59:22,493 DEBUG [RS:0;be1eb2620e0e:34223 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0, corePoolSize=3, maxPoolSize=3 2024-11-20T08:59:22,493 INFO [RS:0;be1eb2620e0e:34223 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-20T08:59:22,494 INFO [RS:0;be1eb2620e0e:34223 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-20T08:59:22,494 INFO [RS:0;be1eb2620e0e:34223 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T08:59:22,494 INFO [RS:0;be1eb2620e0e:34223 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-20T08:59:22,494 INFO [RS:0;be1eb2620e0e:34223 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-20T08:59:22,494 INFO [RS:0;be1eb2620e0e:34223 {}] hbase.ChoreService(168): Chore ScheduledChore name=be1eb2620e0e,34223,1732093159960-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-20T08:59:22,515 INFO [RS:0;be1eb2620e0e:34223 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-20T08:59:22,518 INFO [RS:0;be1eb2620e0e:34223 {}] hbase.ChoreService(168): Chore ScheduledChore name=be1eb2620e0e,34223,1732093159960-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T08:59:22,518 INFO [RS:0;be1eb2620e0e:34223 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T08:59:22,518 INFO [RS:0;be1eb2620e0e:34223 {}] regionserver.Replication(171): be1eb2620e0e,34223,1732093159960 started 2024-11-20T08:59:22,538 INFO [RS:0;be1eb2620e0e:34223 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T08:59:22,539 INFO [RS:0;be1eb2620e0e:34223 {}] regionserver.HRegionServer(1482): Serving as be1eb2620e0e,34223,1732093159960, RpcServer on be1eb2620e0e/172.17.0.2:34223, sessionid=0x1001427dc570001 2024-11-20T08:59:22,540 DEBUG [RS:0;be1eb2620e0e:34223 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-20T08:59:22,540 DEBUG [RS:0;be1eb2620e0e:34223 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:22,541 DEBUG [RS:0;be1eb2620e0e:34223 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'be1eb2620e0e,34223,1732093159960' 2024-11-20T08:59:22,541 DEBUG [RS:0;be1eb2620e0e:34223 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-20T08:59:22,542 DEBUG [RS:0;be1eb2620e0e:34223 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-20T08:59:22,543 DEBUG [RS:0;be1eb2620e0e:34223 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-20T08:59:22,543 DEBUG [RS:0;be1eb2620e0e:34223 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-20T08:59:22,543 DEBUG [RS:0;be1eb2620e0e:34223 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:22,544 DEBUG [RS:0;be1eb2620e0e:34223 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'be1eb2620e0e,34223,1732093159960' 2024-11-20T08:59:22,544 DEBUG [RS:0;be1eb2620e0e:34223 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-20T08:59:22,545 DEBUG [RS:0;be1eb2620e0e:34223 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-20T08:59:22,545 DEBUG [RS:0;be1eb2620e0e:34223 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-20T08:59:22,546 INFO [RS:0;be1eb2620e0e:34223 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-20T08:59:22,546 INFO [RS:0;be1eb2620e0e:34223 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-20T08:59:22,582 WARN [be1eb2620e0e:44373 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-20T08:59:22,652 INFO [RS:0;be1eb2620e0e:34223 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-20T08:59:22,656 INFO [RS:0;be1eb2620e0e:34223 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=be1eb2620e0e%2C34223%2C1732093159960, suffix=, logDir=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/WALs/be1eb2620e0e,34223,1732093159960, archiveDir=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/oldWALs, maxLogs=32 2024-11-20T08:59:22,675 DEBUG [RS:0;be1eb2620e0e:34223 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/WALs/be1eb2620e0e,34223,1732093159960/be1eb2620e0e%2C34223%2C1732093159960.1732093162659, exclude list is [], retry=0 2024-11-20T08:59:22,680 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40033,DS-e81a7599-9758-427f-a85e-bc7af53a6144,DISK] 2024-11-20T08:59:22,684 INFO [RS:0;be1eb2620e0e:34223 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/WALs/be1eb2620e0e,34223,1732093159960/be1eb2620e0e%2C34223%2C1732093159960.1732093162659 2024-11-20T08:59:22,685 DEBUG [RS:0;be1eb2620e0e:34223 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33263:33263)] 2024-11-20T08:59:22,835 DEBUG [be1eb2620e0e:44373 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-20T08:59:22,848 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:22,856 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as be1eb2620e0e,34223,1732093159960, state=OPENING 2024-11-20T08:59:22,862 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-20T08:59:22,865 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44373-0x1001427dc570000, quorum=127.0.0.1:52155, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T08:59:22,865 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34223-0x1001427dc570001, quorum=127.0.0.1:52155, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T08:59:22,866 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T08:59:22,866 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T08:59:22,868 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-20T08:59:22,870 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=be1eb2620e0e,34223,1732093159960}] 2024-11-20T08:59:23,052 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-20T08:59:23,056 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56477, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-20T08:59:23,068 INFO [RS_OPEN_META-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-20T08:59:23,069 INFO [RS_OPEN_META-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T08:59:23,069 INFO [RS_OPEN_META-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-20T08:59:23,073 INFO [RS_OPEN_META-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=be1eb2620e0e%2C34223%2C1732093159960.meta, suffix=.meta, logDir=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/WALs/be1eb2620e0e,34223,1732093159960, archiveDir=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/oldWALs, maxLogs=32 2024-11-20T08:59:23,091 DEBUG [RS_OPEN_META-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/WALs/be1eb2620e0e,34223,1732093159960/be1eb2620e0e%2C34223%2C1732093159960.meta.1732093163074.meta, exclude list is [], retry=0 2024-11-20T08:59:23,095 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40033,DS-e81a7599-9758-427f-a85e-bc7af53a6144,DISK] 2024-11-20T08:59:23,101 INFO [RS_OPEN_META-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/WALs/be1eb2620e0e,34223,1732093159960/be1eb2620e0e%2C34223%2C1732093159960.meta.1732093163074.meta 2024-11-20T08:59:23,101 DEBUG [RS_OPEN_META-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33263:33263)] 2024-11-20T08:59:23,102 DEBUG [RS_OPEN_META-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-20T08:59:23,104 DEBUG [RS_OPEN_META-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-20T08:59:23,107 DEBUG [RS_OPEN_META-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-20T08:59:23,112 INFO [RS_OPEN_META-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-20T08:59:23,116 DEBUG [RS_OPEN_META-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-20T08:59:23,117 DEBUG [RS_OPEN_META-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T08:59:23,117 DEBUG [RS_OPEN_META-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-20T08:59:23,117 DEBUG [RS_OPEN_META-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-20T08:59:23,121 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-20T08:59:23,123 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-20T08:59:23,123 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T08:59:23,124 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T08:59:23,124 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-20T08:59:23,126 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-20T08:59:23,126 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T08:59:23,127 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T08:59:23,127 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-20T08:59:23,129 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-20T08:59:23,129 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T08:59:23,130 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T08:59:23,130 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-20T08:59:23,131 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-20T08:59:23,131 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T08:59:23,132 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T08:59:23,133 DEBUG [RS_OPEN_META-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-20T08:59:23,134 DEBUG [RS_OPEN_META-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/hbase/meta/1588230740 2024-11-20T08:59:23,137 DEBUG [RS_OPEN_META-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/hbase/meta/1588230740 2024-11-20T08:59:23,140 DEBUG [RS_OPEN_META-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-20T08:59:23,140 DEBUG [RS_OPEN_META-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-20T08:59:23,141 DEBUG [RS_OPEN_META-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-20T08:59:23,144 DEBUG [RS_OPEN_META-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-20T08:59:23,146 INFO [RS_OPEN_META-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69840835, jitterRate=0.040709540247917175}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-20T08:59:23,146 DEBUG [RS_OPEN_META-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-20T08:59:23,148 DEBUG [RS_OPEN_META-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732093163118Writing region info on filesystem at 1732093163118Initializing all the Stores at 1732093163120 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732093163120Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732093163121 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732093163121Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732093163121Cleaning up temporary data from old regions at 1732093163140 (+19 ms)Running coprocessor post-open hooks at 1732093163146 (+6 ms)Region opened successfully at 1732093163148 (+2 ms) 2024-11-20T08:59:23,156 INFO [RS_OPEN_META-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732093163041 2024-11-20T08:59:23,169 DEBUG [RS_OPEN_META-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-20T08:59:23,170 INFO [RS_OPEN_META-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-20T08:59:23,171 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:23,174 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as be1eb2620e0e,34223,1732093159960, state=OPEN 2024-11-20T08:59:23,181 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44373-0x1001427dc570000, quorum=127.0.0.1:52155, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T08:59:23,181 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34223-0x1001427dc570001, quorum=127.0.0.1:52155, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T08:59:23,181 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T08:59:23,181 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T08:59:23,182 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:23,187 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-20T08:59:23,188 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=be1eb2620e0e,34223,1732093159960 in 312 msec 2024-11-20T08:59:23,195 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-20T08:59:23,195 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 772 msec 2024-11-20T08:59:23,196 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-20T08:59:23,197 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-20T08:59:23,218 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T08:59:23,219 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=be1eb2620e0e,34223,1732093159960, seqNum=-1] 2024-11-20T08:59:23,244 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T08:59:23,246 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56001, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T08:59:23,269 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.0610 sec 2024-11-20T08:59:23,270 INFO [master/be1eb2620e0e:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732093163270, completionTime=-1 2024-11-20T08:59:23,273 INFO [master/be1eb2620e0e:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-20T08:59:23,273 DEBUG [master/be1eb2620e0e:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-20T08:59:23,304 INFO [master/be1eb2620e0e:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-20T08:59:23,304 INFO [master/be1eb2620e0e:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732093223304 2024-11-20T08:59:23,304 INFO [master/be1eb2620e0e:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732093283304 2024-11-20T08:59:23,304 INFO [master/be1eb2620e0e:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 30 msec 2024-11-20T08:59:23,306 INFO [master/be1eb2620e0e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=be1eb2620e0e,44373,1732093159150-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T08:59:23,307 INFO [master/be1eb2620e0e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=be1eb2620e0e,44373,1732093159150-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T08:59:23,307 INFO [master/be1eb2620e0e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=be1eb2620e0e,44373,1732093159150-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T08:59:23,309 INFO [master/be1eb2620e0e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-be1eb2620e0e:44373, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T08:59:23,309 INFO [master/be1eb2620e0e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-20T08:59:23,310 INFO [master/be1eb2620e0e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-20T08:59:23,316 DEBUG [master/be1eb2620e0e:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-20T08:59:23,348 INFO [master/be1eb2620e0e:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 3.227sec 2024-11-20T08:59:23,350 INFO [master/be1eb2620e0e:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-20T08:59:23,351 INFO [master/be1eb2620e0e:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-20T08:59:23,352 INFO [master/be1eb2620e0e:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-20T08:59:23,353 INFO [master/be1eb2620e0e:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-20T08:59:23,353 INFO [master/be1eb2620e0e:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-20T08:59:23,353 INFO [master/be1eb2620e0e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=be1eb2620e0e,44373,1732093159150-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-20T08:59:23,354 INFO [master/be1eb2620e0e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=be1eb2620e0e,44373,1732093159150-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-20T08:59:23,364 DEBUG [master/be1eb2620e0e:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-20T08:59:23,365 INFO [master/be1eb2620e0e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-20T08:59:23,365 INFO [master/be1eb2620e0e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=be1eb2620e0e,44373,1732093159150-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T08:59:23,382 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@343aff6a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T08:59:23,384 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-20T08:59:23,384 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-20T08:59:23,387 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request be1eb2620e0e,44373,-1 for getting cluster id 2024-11-20T08:59:23,391 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T08:59:23,399 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'caf50035-b19e-4cfd-90ea-dca559866882' 2024-11-20T08:59:23,402 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T08:59:23,402 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "caf50035-b19e-4cfd-90ea-dca559866882" 2024-11-20T08:59:23,403 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5803a924, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T08:59:23,403 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [be1eb2620e0e,44373,-1] 2024-11-20T08:59:23,405 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T08:59:23,407 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T08:59:23,409 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52188, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T08:59:23,412 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32372268, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T08:59:23,413 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T08:59:23,421 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=be1eb2620e0e,34223,1732093159960, seqNum=-1] 2024-11-20T08:59:23,422 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T08:59:23,425 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43570, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T08:59:23,450 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=be1eb2620e0e,44373,1732093159150 2024-11-20T08:59:23,464 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMixedAtomicity Thread=210, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=257, ProcessCount=12, AvailableMemoryMB=2930 2024-11-20T08:59:23,475 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-20T08:59:23,505 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is be1eb2620e0e,44373,1732093159150 2024-11-20T08:59:23,508 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@5164c17f 2024-11-20T08:59:23,510 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-20T08:59:23,516 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52194, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-20T08:59:23,519 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-20T08:59:23,528 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-20T08:59:23,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-20T08:59:23,540 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-20T08:59:23,543 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 4 2024-11-20T08:59:23,543 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T08:59:23,546 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-20T08:59:23,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-20T08:59:23,562 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073741835_1011 (size=959) 2024-11-20T08:59:23,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-20T08:59:23,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-20T08:59:23,966 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 1c52b94a9e6adce80d2d034c31b67cfb, NAME => 'TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a 2024-11-20T08:59:23,977 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073741836_1012 (size=53) 2024-11-20T08:59:24,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-20T08:59:24,379 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T08:59:24,379 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 1c52b94a9e6adce80d2d034c31b67cfb, disabling compactions & flushes 2024-11-20T08:59:24,379 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb. 2024-11-20T08:59:24,379 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb. 2024-11-20T08:59:24,379 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb. after waiting 0 ms 2024-11-20T08:59:24,379 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb. 2024-11-20T08:59:24,380 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb. 2024-11-20T08:59:24,380 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 1c52b94a9e6adce80d2d034c31b67cfb: Waiting for close lock at 1732093164379Disabling compacts and flushes for region at 1732093164379Disabling writes for close at 1732093164379Writing region close event to WAL at 1732093164380 (+1 ms)Closed at 1732093164380 2024-11-20T08:59:24,383 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-20T08:59:24,389 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732093164383"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732093164383"}]},"ts":"1732093164383"} 2024-11-20T08:59:24,394 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-20T08:59:24,396 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-20T08:59:24,399 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732093164396"}]},"ts":"1732093164396"} 2024-11-20T08:59:24,404 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-20T08:59:24,406 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1c52b94a9e6adce80d2d034c31b67cfb, ASSIGN}] 2024-11-20T08:59:24,408 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1c52b94a9e6adce80d2d034c31b67cfb, ASSIGN 2024-11-20T08:59:24,410 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=1c52b94a9e6adce80d2d034c31b67cfb, ASSIGN; state=OFFLINE, location=be1eb2620e0e,34223,1732093159960; forceNewPlan=false, retain=false 2024-11-20T08:59:24,562 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=1c52b94a9e6adce80d2d034c31b67cfb, regionState=OPENING, regionLocation=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:24,567 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=1c52b94a9e6adce80d2d034c31b67cfb, ASSIGN because future has completed 2024-11-20T08:59:24,569 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960}] 2024-11-20T08:59:24,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-20T08:59:24,730 INFO [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb. 2024-11-20T08:59:24,731 DEBUG [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 1c52b94a9e6adce80d2d034c31b67cfb, NAME => 'TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.', STARTKEY => '', ENDKEY => ''} 2024-11-20T08:59:24,731 DEBUG [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 1c52b94a9e6adce80d2d034c31b67cfb 2024-11-20T08:59:24,731 DEBUG [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T08:59:24,732 DEBUG [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 1c52b94a9e6adce80d2d034c31b67cfb 2024-11-20T08:59:24,732 DEBUG [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 1c52b94a9e6adce80d2d034c31b67cfb 2024-11-20T08:59:24,734 INFO [StoreOpener-1c52b94a9e6adce80d2d034c31b67cfb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 1c52b94a9e6adce80d2d034c31b67cfb 2024-11-20T08:59:24,737 INFO [StoreOpener-1c52b94a9e6adce80d2d034c31b67cfb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1c52b94a9e6adce80d2d034c31b67cfb columnFamilyName A 2024-11-20T08:59:24,737 DEBUG [StoreOpener-1c52b94a9e6adce80d2d034c31b67cfb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T08:59:24,738 INFO [StoreOpener-1c52b94a9e6adce80d2d034c31b67cfb-1 {}] regionserver.HStore(327): Store=1c52b94a9e6adce80d2d034c31b67cfb/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T08:59:24,739 INFO [StoreOpener-1c52b94a9e6adce80d2d034c31b67cfb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 1c52b94a9e6adce80d2d034c31b67cfb 2024-11-20T08:59:24,741 INFO [StoreOpener-1c52b94a9e6adce80d2d034c31b67cfb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1c52b94a9e6adce80d2d034c31b67cfb columnFamilyName B 2024-11-20T08:59:24,741 DEBUG [StoreOpener-1c52b94a9e6adce80d2d034c31b67cfb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T08:59:24,742 INFO [StoreOpener-1c52b94a9e6adce80d2d034c31b67cfb-1 {}] regionserver.HStore(327): Store=1c52b94a9e6adce80d2d034c31b67cfb/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T08:59:24,742 INFO [StoreOpener-1c52b94a9e6adce80d2d034c31b67cfb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 1c52b94a9e6adce80d2d034c31b67cfb 2024-11-20T08:59:24,744 INFO [StoreOpener-1c52b94a9e6adce80d2d034c31b67cfb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1c52b94a9e6adce80d2d034c31b67cfb columnFamilyName C 2024-11-20T08:59:24,744 DEBUG [StoreOpener-1c52b94a9e6adce80d2d034c31b67cfb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T08:59:24,745 INFO [StoreOpener-1c52b94a9e6adce80d2d034c31b67cfb-1 {}] regionserver.HStore(327): Store=1c52b94a9e6adce80d2d034c31b67cfb/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T08:59:24,745 DEBUG [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 1c52b94a9e6adce80d2d034c31b67cfb 2024-11-20T08:59:24,746 DEBUG [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb 2024-11-20T08:59:24,747 DEBUG [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb 2024-11-20T08:59:24,749 DEBUG [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 1c52b94a9e6adce80d2d034c31b67cfb 2024-11-20T08:59:24,749 DEBUG [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 1c52b94a9e6adce80d2d034c31b67cfb 2024-11-20T08:59:24,750 DEBUG [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-20T08:59:24,752 DEBUG [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 1c52b94a9e6adce80d2d034c31b67cfb 2024-11-20T08:59:24,757 DEBUG [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T08:59:24,758 INFO [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 1c52b94a9e6adce80d2d034c31b67cfb; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65030871, jitterRate=-0.030964508652687073}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-20T08:59:24,758 DEBUG [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1c52b94a9e6adce80d2d034c31b67cfb 2024-11-20T08:59:24,759 DEBUG [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 1c52b94a9e6adce80d2d034c31b67cfb: Running coprocessor pre-open hook at 1732093164732Writing region info on filesystem at 1732093164732Initializing all the Stores at 1732093164734 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732093164734Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732093164734Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732093164734Cleaning up temporary data from old regions at 1732093164749 (+15 ms)Running coprocessor post-open hooks at 1732093164758 (+9 ms)Region opened successfully at 1732093164759 (+1 ms) 2024-11-20T08:59:24,762 INFO [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., pid=6, masterSystemTime=1732093164723 2024-11-20T08:59:24,766 DEBUG [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb. 2024-11-20T08:59:24,766 INFO [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb. 2024-11-20T08:59:24,767 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=1c52b94a9e6adce80d2d034c31b67cfb, regionState=OPEN, openSeqNum=2, regionLocation=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:24,771 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 because future has completed 2024-11-20T08:59:24,779 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-20T08:59:24,779 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 in 206 msec 2024-11-20T08:59:24,783 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-20T08:59:24,783 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1c52b94a9e6adce80d2d034c31b67cfb, ASSIGN in 373 msec 2024-11-20T08:59:24,785 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-20T08:59:24,786 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732093164785"}]},"ts":"1732093164785"} 2024-11-20T08:59:24,789 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-20T08:59:24,791 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-20T08:59:24,795 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.2600 sec 2024-11-20T08:59:25,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-20T08:59:25,699 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-20T08:59:25,705 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f0b3b59, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T08:59:25,705 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request be1eb2620e0e,44373,-1 for getting cluster id 2024-11-20T08:59:25,706 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T08:59:25,708 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'caf50035-b19e-4cfd-90ea-dca559866882' 2024-11-20T08:59:25,709 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T08:59:25,709 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "caf50035-b19e-4cfd-90ea-dca559866882" 2024-11-20T08:59:25,709 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7983b71f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T08:59:25,709 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [be1eb2620e0e,44373,-1] 2024-11-20T08:59:25,710 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T08:59:25,710 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T08:59:25,711 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52220, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T08:59:25,713 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e34b7e6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T08:59:25,713 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T08:59:25,715 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=be1eb2620e0e,34223,1732093159960, seqNum=-1] 2024-11-20T08:59:25,715 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T08:59:25,717 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43584, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T08:59:25,720 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-20T08:59:25,721 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is be1eb2620e0e,44373,1732093159150 2024-11-20T08:59:25,721 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@2d5e92ab 2024-11-20T08:59:25,724 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-20T08:59:25,726 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52224, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-20T08:59:25,731 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@340b46e5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T08:59:25,731 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request be1eb2620e0e,44373,-1 for getting cluster id 2024-11-20T08:59:25,731 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T08:59:25,733 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'caf50035-b19e-4cfd-90ea-dca559866882' 2024-11-20T08:59:25,733 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T08:59:25,733 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "caf50035-b19e-4cfd-90ea-dca559866882" 2024-11-20T08:59:25,734 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@175526a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T08:59:25,734 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [be1eb2620e0e,44373,-1] 2024-11-20T08:59:25,734 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T08:59:25,735 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T08:59:25,736 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52232, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T08:59:25,737 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f55d533, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T08:59:25,738 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6edf4fef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T08:59:25,738 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request be1eb2620e0e,44373,-1 for getting cluster id 2024-11-20T08:59:25,738 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T08:59:25,740 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'caf50035-b19e-4cfd-90ea-dca559866882' 2024-11-20T08:59:25,740 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T08:59:25,740 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "caf50035-b19e-4cfd-90ea-dca559866882" 2024-11-20T08:59:25,741 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ccb4eda, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T08:59:25,741 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [be1eb2620e0e,44373,-1] 2024-11-20T08:59:25,741 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T08:59:25,742 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T08:59:25,743 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52258, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T08:59:25,745 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@fc8436a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T08:59:25,745 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21f4b80, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T08:59:25,745 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request be1eb2620e0e,44373,-1 for getting cluster id 2024-11-20T08:59:25,746 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T08:59:25,748 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'caf50035-b19e-4cfd-90ea-dca559866882' 2024-11-20T08:59:25,749 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T08:59:25,749 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "caf50035-b19e-4cfd-90ea-dca559866882" 2024-11-20T08:59:25,750 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10bc6ecb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T08:59:25,750 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [be1eb2620e0e,44373,-1] 2024-11-20T08:59:25,750 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T08:59:25,751 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T08:59:25,752 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52266, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T08:59:25,753 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64257f01, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T08:59:25,754 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52294ca5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T08:59:25,754 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request be1eb2620e0e,44373,-1 for getting cluster id 2024-11-20T08:59:25,754 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T08:59:25,756 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'caf50035-b19e-4cfd-90ea-dca559866882' 2024-11-20T08:59:25,756 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T08:59:25,757 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "caf50035-b19e-4cfd-90ea-dca559866882" 2024-11-20T08:59:25,757 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a27ead7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T08:59:25,757 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [be1eb2620e0e,44373,-1] 2024-11-20T08:59:25,757 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T08:59:25,760 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T08:59:25,762 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52282, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T08:59:25,763 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@457a3094, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T08:59:25,766 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d7926de, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T08:59:25,766 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request be1eb2620e0e,44373,-1 for getting cluster id 2024-11-20T08:59:25,766 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T08:59:25,767 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'caf50035-b19e-4cfd-90ea-dca559866882' 2024-11-20T08:59:25,767 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T08:59:25,768 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "caf50035-b19e-4cfd-90ea-dca559866882" 2024-11-20T08:59:25,768 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f99b884, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T08:59:25,768 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [be1eb2620e0e,44373,-1] 2024-11-20T08:59:25,769 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T08:59:25,769 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T08:59:25,770 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52302, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T08:59:25,771 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@75abeb9b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T08:59:25,773 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b48425, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T08:59:25,773 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request be1eb2620e0e,44373,-1 for getting cluster id 2024-11-20T08:59:25,773 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T08:59:25,774 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'caf50035-b19e-4cfd-90ea-dca559866882' 2024-11-20T08:59:25,774 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T08:59:25,774 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "caf50035-b19e-4cfd-90ea-dca559866882" 2024-11-20T08:59:25,775 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@598c3d89, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T08:59:25,775 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [be1eb2620e0e,44373,-1] 2024-11-20T08:59:25,776 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T08:59:25,776 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T08:59:25,777 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52324, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T08:59:25,778 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49f892ed, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T08:59:25,779 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4454ce2a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T08:59:25,779 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request be1eb2620e0e,44373,-1 for getting cluster id 2024-11-20T08:59:25,779 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T08:59:25,780 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'caf50035-b19e-4cfd-90ea-dca559866882' 2024-11-20T08:59:25,780 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T08:59:25,780 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "caf50035-b19e-4cfd-90ea-dca559866882" 2024-11-20T08:59:25,781 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27d22a9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T08:59:25,781 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [be1eb2620e0e,44373,-1] 2024-11-20T08:59:25,781 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T08:59:25,782 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T08:59:25,782 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52340, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T08:59:25,783 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@545d818d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T08:59:25,788 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74a9c1c7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T08:59:25,788 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request be1eb2620e0e,44373,-1 for getting cluster id 2024-11-20T08:59:25,788 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T08:59:25,789 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'caf50035-b19e-4cfd-90ea-dca559866882' 2024-11-20T08:59:25,790 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T08:59:25,790 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "caf50035-b19e-4cfd-90ea-dca559866882" 2024-11-20T08:59:25,790 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@783f9474, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T08:59:25,790 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [be1eb2620e0e,44373,-1] 2024-11-20T08:59:25,791 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T08:59:25,791 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T08:59:25,792 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52354, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T08:59:25,793 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@38e858d9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T08:59:25,794 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1cadbd93, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T08:59:25,794 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request be1eb2620e0e,44373,-1 for getting cluster id 2024-11-20T08:59:25,794 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T08:59:25,795 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'caf50035-b19e-4cfd-90ea-dca559866882' 2024-11-20T08:59:25,795 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T08:59:25,795 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "caf50035-b19e-4cfd-90ea-dca559866882" 2024-11-20T08:59:25,795 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21f437cd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T08:59:25,795 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [be1eb2620e0e,44373,-1] 2024-11-20T08:59:25,796 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T08:59:25,796 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T08:59:25,797 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52364, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T08:59:25,798 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a903853, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T08:59:25,801 DEBUG [Thread-160 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T08:59:25,803 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=be1eb2620e0e,34223,1732093159960, seqNum=-1] 2024-11-20T08:59:25,803 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T08:59:25,804 DEBUG [Thread-154 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T08:59:25,805 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43586, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T08:59:25,806 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=be1eb2620e0e,34223,1732093159960, seqNum=-1] 2024-11-20T08:59:25,806 DEBUG [Thread-149 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T08:59:25,807 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T08:59:25,807 DEBUG [Thread-147 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T08:59:25,808 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43602, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T08:59:25,808 DEBUG [Thread-149 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=be1eb2620e0e,34223,1732093159960, seqNum=-1] 2024-11-20T08:59:25,809 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=be1eb2620e0e,34223,1732093159960, seqNum=-1] 2024-11-20T08:59:25,809 DEBUG [Thread-149 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T08:59:25,809 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T08:59:25,810 DEBUG [Thread-151 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T08:59:25,811 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43608, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T08:59:25,812 DEBUG [Thread-156 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T08:59:25,812 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43620, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T08:59:25,812 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=be1eb2620e0e,34223,1732093159960, seqNum=-1] 2024-11-20T08:59:25,812 DEBUG [Thread-143 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T08:59:25,812 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T08:59:25,814 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=be1eb2620e0e,34223,1732093159960, seqNum=-1] 2024-11-20T08:59:25,814 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T08:59:25,815 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43628, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T08:59:25,816 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=be1eb2620e0e,34223,1732093159960, seqNum=-1] 2024-11-20T08:59:25,816 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43644, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T08:59:25,816 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T08:59:25,817 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2] 2024-11-20T08:59:25,817 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2] 2024-11-20T08:59:25,819 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43658, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T08:59:25,819 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T08:59:25,819 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2] 2024-11-20T08:59:25,821 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2] 2024-11-20T08:59:25,821 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2] 2024-11-20T08:59:25,823 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2] 2024-11-20T08:59:25,824 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2] 2024-11-20T08:59:25,825 DEBUG [Thread-145 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T08:59:25,829 DEBUG [Thread-158 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T08:59:25,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees 2024-11-20T08:59:25,833 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=be1eb2620e0e,34223,1732093159960, seqNum=-1] 2024-11-20T08:59:25,833 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T08:59:25,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-20T08:59:25,835 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43664, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T08:59:25,836 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=be1eb2620e0e,34223,1732093159960, seqNum=-1] 2024-11-20T08:59:25,836 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T08:59:25,838 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43672, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T08:59:25,842 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2] 2024-11-20T08:59:25,842 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2] 2024-11-20T08:59:25,844 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T08:59:25,847 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T08:59:25,850 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T08:59:25,914 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T08:59:25,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(8855): Flush requested on 1c52b94a9e6adce80d2d034c31b67cfb 2024-11-20T08:59:25,919 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1c52b94a9e6adce80d2d034c31b67cfb 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-20T08:59:25,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-20T08:59:26,019 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-20T08:59:26,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb. 2024-11-20T08:59:26,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb. as already flushing 2024-11-20T08:59:26,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb. 2024-11-20T08:59:26,038 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 1c52b94a9e6adce80d2d034c31b67cfb, NAME => 'TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T08:59:26,052 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/A/18e402d32c8949b7836f0c78adf42ea6 is 50, key is test_row_0/A:col10/1732093165912/Put/seqid=0 2024-11-20T08:59:26,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 1c52b94a9e6adce80d2d034c31b67cfb, NAME => 'TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T08:59:26,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1c52b94a9e6adce80d2d034c31b67cfb, NAME => 'TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1c52b94a9e6adce80d2d034c31b67cfb, NAME => 'TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T08:59:26,080 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073741837_1013 (size=12001) 2024-11-20T08:59:26,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-20T08:59:26,185 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:26,186 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:26,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43628 deadline: 1732093226167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:26,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43620 deadline: 1732093226170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:26,193 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:26,193 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:26,194 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:26,194 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:26,194 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:26,194 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:26,197 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:26,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43672 deadline: 1732093226187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:26,200 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:26,200 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:26,200 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:26,200 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:26,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43608 deadline: 1732093226188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:26,204 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:26,204 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:26,204 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:26,204 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:26,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43644 deadline: 1732093226191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:26,206 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:26,206 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:26,206 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:26,222 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-20T08:59:26,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb. 2024-11-20T08:59:26,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb. as already flushing 2024-11-20T08:59:26,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb. 2024-11-20T08:59:26,227 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 1c52b94a9e6adce80d2d034c31b67cfb, NAME => 'TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T08:59:26,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 1c52b94a9e6adce80d2d034c31b67cfb, NAME => 'TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T08:59:26,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1c52b94a9e6adce80d2d034c31b67cfb, NAME => 'TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1c52b94a9e6adce80d2d034c31b67cfb, NAME => 'TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T08:59:26,306 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:26,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43620 deadline: 1732093226298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:26,308 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:26,308 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:26,308 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:26,312 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:26,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43628 deadline: 1732093226307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:26,313 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:26,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43672 deadline: 1732093226308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:26,314 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:26,315 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:26,315 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:26,315 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:26,315 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:26,315 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:26,315 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:26,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43608 deadline: 1732093226309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:26,316 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:26,316 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:26,317 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:26,320 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:26,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43644 deadline: 1732093226318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:26,321 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:26,321 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:26,321 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:26,384 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-20T08:59:26,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb. 2024-11-20T08:59:26,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb. as already flushing 2024-11-20T08:59:26,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb. 2024-11-20T08:59:26,394 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 1c52b94a9e6adce80d2d034c31b67cfb, NAME => 'TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T08:59:26,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 1c52b94a9e6adce80d2d034c31b67cfb, NAME => 'TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T08:59:26,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1c52b94a9e6adce80d2d034c31b67cfb, NAME => 'TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1c52b94a9e6adce80d2d034c31b67cfb, NAME => 'TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T08:59:26,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-20T08:59:26,482 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/A/18e402d32c8949b7836f0c78adf42ea6 2024-11-20T08:59:26,522 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:26,522 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:26,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43672 deadline: 1732093226520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:26,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43620 deadline: 1732093226520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:26,524 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:26,524 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:26,524 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:26,524 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:26,524 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:26,524 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:26,534 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:26,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43628 deadline: 1732093226528, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:26,535 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:26,536 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:26,536 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:26,537 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:26,537 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:26,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43608 deadline: 1732093226530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:26,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43644 deadline: 1732093226531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:26,538 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:26,538 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:26,538 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:26,539 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:26,539 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:26,539 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:26,550 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-20T08:59:26,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb. 2024-11-20T08:59:26,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb. as already flushing 2024-11-20T08:59:26,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb. 2024-11-20T08:59:26,550 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 1c52b94a9e6adce80d2d034c31b67cfb, NAME => 'TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T08:59:26,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 1c52b94a9e6adce80d2d034c31b67cfb, NAME => 'TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T08:59:26,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1c52b94a9e6adce80d2d034c31b67cfb, NAME => 'TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1c52b94a9e6adce80d2d034c31b67cfb, NAME => 'TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T08:59:26,604 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/B/41c599c3b8ac47f291b46a98be3b0adf is 50, key is test_row_0/B:col10/1732093165912/Put/seqid=0 2024-11-20T08:59:26,628 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073741838_1014 (size=12001) 2024-11-20T08:59:26,634 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/B/41c599c3b8ac47f291b46a98be3b0adf 2024-11-20T08:59:26,687 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/C/8af8ce1ea67945a7a89b77ec26594a49 is 50, key is test_row_0/C:col10/1732093165912/Put/seqid=0 2024-11-20T08:59:26,704 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073741839_1015 (size=12001) 2024-11-20T08:59:26,705 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/C/8af8ce1ea67945a7a89b77ec26594a49 2024-11-20T08:59:26,707 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-20T08:59:26,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb. 2024-11-20T08:59:26,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb. as already flushing 2024-11-20T08:59:26,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb. 2024-11-20T08:59:26,708 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 1c52b94a9e6adce80d2d034c31b67cfb, NAME => 'TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T08:59:26,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 1c52b94a9e6adce80d2d034c31b67cfb, NAME => 'TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T08:59:26,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1c52b94a9e6adce80d2d034c31b67cfb, NAME => 'TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1c52b94a9e6adce80d2d034c31b67cfb, NAME => 'TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T08:59:26,722 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/A/18e402d32c8949b7836f0c78adf42ea6 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/18e402d32c8949b7836f0c78adf42ea6 2024-11-20T08:59:26,738 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/18e402d32c8949b7836f0c78adf42ea6, entries=150, sequenceid=13, filesize=11.7 K 2024-11-20T08:59:26,743 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/B/41c599c3b8ac47f291b46a98be3b0adf as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/41c599c3b8ac47f291b46a98be3b0adf 2024-11-20T08:59:26,758 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/41c599c3b8ac47f291b46a98be3b0adf, entries=150, sequenceid=13, filesize=11.7 K 2024-11-20T08:59:26,761 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/C/8af8ce1ea67945a7a89b77ec26594a49 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/C/8af8ce1ea67945a7a89b77ec26594a49 2024-11-20T08:59:26,782 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/C/8af8ce1ea67945a7a89b77ec26594a49, entries=150, sequenceid=13, filesize=11.7 K 2024-11-20T08:59:26,796 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=154.31 KB/158010 for 1c52b94a9e6adce80d2d034c31b67cfb in 874ms, sequenceid=13, compaction requested=false 2024-11-20T08:59:26,798 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-20T08:59:26,801 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1c52b94a9e6adce80d2d034c31b67cfb: 2024-11-20T08:59:26,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(8855): Flush requested on 1c52b94a9e6adce80d2d034c31b67cfb 2024-11-20T08:59:26,849 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T08:59:26,849 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1c52b94a9e6adce80d2d034c31b67cfb 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-20T08:59:26,865 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-20T08:59:26,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb. 2024-11-20T08:59:26,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb. as already flushing 2024-11-20T08:59:26,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb. 2024-11-20T08:59:26,866 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 1c52b94a9e6adce80d2d034c31b67cfb, NAME => 'TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T08:59:26,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 1c52b94a9e6adce80d2d034c31b67cfb, NAME => 'TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T08:59:26,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1c52b94a9e6adce80d2d034c31b67cfb, NAME => 'TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1c52b94a9e6adce80d2d034c31b67cfb, NAME => 'TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T08:59:26,875 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/A/5dad489f7b8e471988ef015ceab18ecf is 50, key is test_row_0/A:col10/1732093166180/Put/seqid=0 2024-11-20T08:59:26,900 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073741840_1016 (size=14341) 2024-11-20T08:59:26,904 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/A/5dad489f7b8e471988ef015ceab18ecf 2024-11-20T08:59:26,907 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:26,906 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:26,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43644 deadline: 1732093226884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:26,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43672 deadline: 1732093226883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:26,909 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:26,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43628 deadline: 1732093226888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:26,910 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:26,910 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:26,910 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:26,910 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:26,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43608 deadline: 1732093226888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:26,912 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:26,912 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:26,912 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:26,912 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:26,912 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:26,912 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:26,912 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:26,912 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:26,912 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:26,933 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:26,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43620 deadline: 1732093226907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:26,934 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:26,934 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:26,934 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:26,935 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/B/03f8e222cd39437caba4b06e0e17c2eb is 50, key is test_row_0/B:col10/1732093166180/Put/seqid=0 2024-11-20T08:59:26,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-20T08:59:26,976 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073741841_1017 (size=12001) 2024-11-20T08:59:26,977 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/B/03f8e222cd39437caba4b06e0e17c2eb 2024-11-20T08:59:27,012 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/C/48fc85d6826d4311b60e6d880cd48696 is 50, key is test_row_0/C:col10/1732093166180/Put/seqid=0 2024-11-20T08:59:27,021 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-20T08:59:27,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb. 2024-11-20T08:59:27,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb. as already flushing 2024-11-20T08:59:27,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb. 2024-11-20T08:59:27,022 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 1c52b94a9e6adce80d2d034c31b67cfb, NAME => 'TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T08:59:27,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 1c52b94a9e6adce80d2d034c31b67cfb, NAME => 'TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T08:59:27,023 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:27,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43628 deadline: 1732093227018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:27,024 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:27,024 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:27,024 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:27,024 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:27,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43644 deadline: 1732093227020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:27,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1c52b94a9e6adce80d2d034c31b67cfb, NAME => 'TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1c52b94a9e6adce80d2d034c31b67cfb, NAME => 'TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T08:59:27,026 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:27,026 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:27,026 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:27,026 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:27,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43608 deadline: 1732093227020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:27,027 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:27,027 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:27,027 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:27,028 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:27,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43672 deadline: 1732093227021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:27,029 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:27,030 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:27,030 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:27,037 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073741842_1018 (size=12001) 2024-11-20T08:59:27,039 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/C/48fc85d6826d4311b60e6d880cd48696 2024-11-20T08:59:27,054 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/A/5dad489f7b8e471988ef015ceab18ecf as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/5dad489f7b8e471988ef015ceab18ecf 2024-11-20T08:59:27,056 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:27,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43620 deadline: 1732093227038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:27,062 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:27,062 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:27,062 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:27,072 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/5dad489f7b8e471988ef015ceab18ecf, entries=200, sequenceid=40, filesize=14.0 K 2024-11-20T08:59:27,075 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/B/03f8e222cd39437caba4b06e0e17c2eb as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/03f8e222cd39437caba4b06e0e17c2eb 2024-11-20T08:59:27,087 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/03f8e222cd39437caba4b06e0e17c2eb, entries=150, sequenceid=40, filesize=11.7 K 2024-11-20T08:59:27,090 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/C/48fc85d6826d4311b60e6d880cd48696 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/C/48fc85d6826d4311b60e6d880cd48696 2024-11-20T08:59:27,104 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/C/48fc85d6826d4311b60e6d880cd48696, entries=150, sequenceid=40, filesize=11.7 K 2024-11-20T08:59:27,106 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=40.25 KB/41220 for 1c52b94a9e6adce80d2d034c31b67cfb in 256ms, sequenceid=40, compaction requested=false 2024-11-20T08:59:27,106 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1c52b94a9e6adce80d2d034c31b67cfb: 2024-11-20T08:59:27,180 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-20T08:59:27,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb. 2024-11-20T08:59:27,181 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing 1c52b94a9e6adce80d2d034c31b67cfb 3/3 column families, dataSize=40.25 KB heapSize=106.22 KB 2024-11-20T08:59:27,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/A/17bb534da4d54370919a658d6e8821fd is 50, key is test_row_0/A:col10/1732093166901/Put/seqid=0 2024-11-20T08:59:27,228 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073741843_1019 (size=9657) 2024-11-20T08:59:27,230 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/A/17bb534da4d54370919a658d6e8821fd 2024-11-20T08:59:27,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(8855): Flush requested on 1c52b94a9e6adce80d2d034c31b67cfb 2024-11-20T08:59:27,251 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb. as already flushing 2024-11-20T08:59:27,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/B/fc87d3e960fe4f3fba71af7835275984 is 50, key is test_row_0/B:col10/1732093166901/Put/seqid=0 2024-11-20T08:59:27,307 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073741844_1020 (size=9657) 2024-11-20T08:59:27,308 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/B/fc87d3e960fe4f3fba71af7835275984 2024-11-20T08:59:27,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/C/9f71a2dc6b5a4d6781eff85497784247 is 50, key is test_row_0/C:col10/1732093166901/Put/seqid=0 2024-11-20T08:59:27,350 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073741845_1021 (size=9657) 2024-11-20T08:59:27,351 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/C/9f71a2dc6b5a4d6781eff85497784247 2024-11-20T08:59:27,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/A/17bb534da4d54370919a658d6e8821fd as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/17bb534da4d54370919a658d6e8821fd 2024-11-20T08:59:27,376 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/17bb534da4d54370919a658d6e8821fd, entries=100, sequenceid=49, filesize=9.4 K 2024-11-20T08:59:27,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/B/fc87d3e960fe4f3fba71af7835275984 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/fc87d3e960fe4f3fba71af7835275984 2024-11-20T08:59:27,398 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/fc87d3e960fe4f3fba71af7835275984, entries=100, sequenceid=49, filesize=9.4 K 2024-11-20T08:59:27,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/C/9f71a2dc6b5a4d6781eff85497784247 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/C/9f71a2dc6b5a4d6781eff85497784247 2024-11-20T08:59:27,418 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:27,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43672 deadline: 1732093227385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:27,418 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/C/9f71a2dc6b5a4d6781eff85497784247, entries=100, sequenceid=49, filesize=9.4 K 2024-11-20T08:59:27,420 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:27,420 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:27,420 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:27,420 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~106.17 KB/108720, currentSize=167.72 KB/171750 for 1c52b94a9e6adce80d2d034c31b67cfb in 239ms, sequenceid=49, compaction requested=true 2024-11-20T08:59:27,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for 1c52b94a9e6adce80d2d034c31b67cfb: 2024-11-20T08:59:27,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb. 2024-11-20T08:59:27,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-11-20T08:59:27,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-11-20T08:59:27,427 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-20T08:59:27,428 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5730 sec 2024-11-20T08:59:27,432 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees in 1.6080 sec 2024-11-20T08:59:27,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(8855): Flush requested on 1c52b94a9e6adce80d2d034c31b67cfb 2024-11-20T08:59:27,437 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T08:59:27,437 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1c52b94a9e6adce80d2d034c31b67cfb 3/3 column families, dataSize=174.43 KB heapSize=457.78 KB 2024-11-20T08:59:27,456 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/A/676c71b52c0c49268ae13273499e7737 is 50, key is test_row_0/A:col10/1732093167412/Put/seqid=0 2024-11-20T08:59:27,460 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:27,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43620 deadline: 1732093227445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:27,461 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:27,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43608 deadline: 1732093227451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:27,461 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:27,461 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:27,462 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:27,463 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:27,464 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:27,464 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:27,474 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073741846_1022 (size=14341) 2024-11-20T08:59:27,475 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=78 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/A/676c71b52c0c49268ae13273499e7737 2024-11-20T08:59:27,477 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:27,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43628 deadline: 1732093227465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:27,479 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:27,479 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:27,479 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:27,479 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:27,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43644 deadline: 1732093227465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:27,480 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:27,480 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:27,480 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:27,510 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/B/50b9bc81535f4713be9d102b543ca261 is 50, key is test_row_0/B:col10/1732093167412/Put/seqid=0 2024-11-20T08:59:27,533 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073741847_1023 (size=12001) 2024-11-20T08:59:27,535 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:27,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43672 deadline: 1732093227529, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:27,537 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:27,537 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:27,537 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:27,540 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=78 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/B/50b9bc81535f4713be9d102b543ca261 2024-11-20T08:59:27,563 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/C/7b7908ca94e747e9b9eeb1e9db2dc007 is 50, key is test_row_0/C:col10/1732093167412/Put/seqid=0 2024-11-20T08:59:27,569 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:27,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43620 deadline: 1732093227568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:27,571 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:27,571 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:27,571 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:27,571 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:27,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43608 deadline: 1732093227568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:27,572 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:27,572 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:27,572 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:27,595 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073741848_1024 (size=12001) 2024-11-20T08:59:27,597 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=78 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/C/7b7908ca94e747e9b9eeb1e9db2dc007 2024-11-20T08:59:27,601 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:27,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43628 deadline: 1732093227587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:27,602 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:27,602 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:27,602 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:27,603 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:27,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43644 deadline: 1732093227587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:27,604 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:27,604 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:27,604 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:27,616 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/A/676c71b52c0c49268ae13273499e7737 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/676c71b52c0c49268ae13273499e7737 2024-11-20T08:59:27,630 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/676c71b52c0c49268ae13273499e7737, entries=200, sequenceid=78, filesize=14.0 K 2024-11-20T08:59:27,631 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreScanner(1005): StoreScanner already closing. There is no need to updateReaders 2024-11-20T08:59:27,634 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/B/50b9bc81535f4713be9d102b543ca261 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/50b9bc81535f4713be9d102b543ca261 2024-11-20T08:59:27,645 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/50b9bc81535f4713be9d102b543ca261, entries=150, sequenceid=78, filesize=11.7 K 2024-11-20T08:59:27,647 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/C/7b7908ca94e747e9b9eeb1e9db2dc007 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/C/7b7908ca94e747e9b9eeb1e9db2dc007 2024-11-20T08:59:27,660 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/C/7b7908ca94e747e9b9eeb1e9db2dc007, entries=150, sequenceid=78, filesize=11.7 K 2024-11-20T08:59:27,662 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~174.43 KB/178620, heapSize ~457.73 KB/468720, currentSize=33.54 KB/34350 for 1c52b94a9e6adce80d2d034c31b67cfb in 225ms, sequenceid=78, compaction requested=true 2024-11-20T08:59:27,662 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1c52b94a9e6adce80d2d034c31b67cfb: 2024-11-20T08:59:27,683 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1c52b94a9e6adce80d2d034c31b67cfb:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T08:59:27,686 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T08:59:27,687 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T08:59:27,687 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1c52b94a9e6adce80d2d034c31b67cfb:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T08:59:27,687 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T08:59:27,687 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1c52b94a9e6adce80d2d034c31b67cfb:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T08:59:27,687 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-20T08:59:27,687 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T08:59:27,692 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 45660 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T08:59:27,694 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50340 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T08:59:27,694 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1541): 1c52b94a9e6adce80d2d034c31b67cfb/B is initiating minor compaction (all files) 2024-11-20T08:59:27,694 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HStore(1541): 1c52b94a9e6adce80d2d034c31b67cfb/A is initiating minor compaction (all files) 2024-11-20T08:59:27,695 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1c52b94a9e6adce80d2d034c31b67cfb/B in TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb. 2024-11-20T08:59:27,697 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/41c599c3b8ac47f291b46a98be3b0adf, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/03f8e222cd39437caba4b06e0e17c2eb, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/fc87d3e960fe4f3fba71af7835275984, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/50b9bc81535f4713be9d102b543ca261] into tmpdir=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp, totalSize=44.6 K 2024-11-20T08:59:27,698 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1c52b94a9e6adce80d2d034c31b67cfb/A in TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb. 2024-11-20T08:59:27,698 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/18e402d32c8949b7836f0c78adf42ea6, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/5dad489f7b8e471988ef015ceab18ecf, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/17bb534da4d54370919a658d6e8821fd, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/676c71b52c0c49268ae13273499e7737] into tmpdir=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp, totalSize=49.2 K 2024-11-20T08:59:27,699 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting 41c599c3b8ac47f291b46a98be3b0adf, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1732093165879 2024-11-20T08:59:27,699 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.Compactor(225): Compacting 18e402d32c8949b7836f0c78adf42ea6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1732093165879 2024-11-20T08:59:27,699 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting 03f8e222cd39437caba4b06e0e17c2eb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1732093166105 2024-11-20T08:59:27,700 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5dad489f7b8e471988ef015ceab18ecf, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1732093166105 2024-11-20T08:59:27,700 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting fc87d3e960fe4f3fba71af7835275984, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732093166872 2024-11-20T08:59:27,701 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting 50b9bc81535f4713be9d102b543ca261, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1732093167411 2024-11-20T08:59:27,701 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.Compactor(225): Compacting 17bb534da4d54370919a658d6e8821fd, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732093166872 2024-11-20T08:59:27,703 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.Compactor(225): Compacting 676c71b52c0c49268ae13273499e7737, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1732093167411 2024-11-20T08:59:27,748 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1c52b94a9e6adce80d2d034c31b67cfb#B#compaction#12 average throughput is 0.94 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-20T08:59:27,748 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1c52b94a9e6adce80d2d034c31b67cfb#A#compaction#13 average throughput is 0.94 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T08:59:27,749 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/B/a0f5cdf99a1841e6bb7c9d5933220ad1 is 50, key is test_row_0/B:col10/1732093167412/Put/seqid=0 2024-11-20T08:59:27,749 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/A/8a99370765c549529790491b7129bc4b is 50, key is test_row_0/A:col10/1732093167412/Put/seqid=0 2024-11-20T08:59:27,773 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T08:59:27,773 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1c52b94a9e6adce80d2d034c31b67cfb 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-20T08:59:27,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073741849_1025 (size=12139) 2024-11-20T08:59:27,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(8855): Flush requested on 1c52b94a9e6adce80d2d034c31b67cfb 2024-11-20T08:59:27,788 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/A/54173cb23d0b4a6a80f35a7c576c08e3 is 50, key is test_row_0/A:col10/1732093167759/Put/seqid=0 2024-11-20T08:59:27,792 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/B/a0f5cdf99a1841e6bb7c9d5933220ad1 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/a0f5cdf99a1841e6bb7c9d5933220ad1 2024-11-20T08:59:27,798 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073741850_1026 (size=12139) 2024-11-20T08:59:27,827 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073741851_1027 (size=11997) 2024-11-20T08:59:27,838 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1c52b94a9e6adce80d2d034c31b67cfb/B of 1c52b94a9e6adce80d2d034c31b67cfb into a0f5cdf99a1841e6bb7c9d5933220ad1(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T08:59:27,838 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1c52b94a9e6adce80d2d034c31b67cfb: 2024-11-20T08:59:27,841 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., storeName=1c52b94a9e6adce80d2d034c31b67cfb/B, priority=12, startTime=1732093167687; duration=0sec 2024-11-20T08:59:27,842 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T08:59:27,842 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1c52b94a9e6adce80d2d034c31b67cfb:B 2024-11-20T08:59:27,842 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T08:59:27,846 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 45660 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T08:59:27,847 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1541): 1c52b94a9e6adce80d2d034c31b67cfb/C is initiating minor compaction (all files) 2024-11-20T08:59:27,847 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1c52b94a9e6adce80d2d034c31b67cfb/C in TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb. 2024-11-20T08:59:27,847 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/C/8af8ce1ea67945a7a89b77ec26594a49, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/C/48fc85d6826d4311b60e6d880cd48696, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/C/9f71a2dc6b5a4d6781eff85497784247, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/C/7b7908ca94e747e9b9eeb1e9db2dc007] into tmpdir=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp, totalSize=44.6 K 2024-11-20T08:59:27,848 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting 8af8ce1ea67945a7a89b77ec26594a49, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1732093165879 2024-11-20T08:59:27,849 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting 48fc85d6826d4311b60e6d880cd48696, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1732093166105 2024-11-20T08:59:27,849 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting 9f71a2dc6b5a4d6781eff85497784247, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732093166872 2024-11-20T08:59:27,850 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting 7b7908ca94e747e9b9eeb1e9db2dc007, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1732093167411 2024-11-20T08:59:27,893 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1c52b94a9e6adce80d2d034c31b67cfb#C#compaction#15 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T08:59:27,894 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/C/86e23a1f4fac4b60baca926bfe626ad0 is 50, key is test_row_0/C:col10/1732093167412/Put/seqid=0 2024-11-20T08:59:27,906 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:27,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43644 deadline: 1732093227894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:27,908 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:27,908 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:27,908 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:27,915 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:27,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43620 deadline: 1732093227897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:27,917 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:27,917 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:27,917 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:27,921 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073741852_1028 (size=12139) 2024-11-20T08:59:27,922 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:27,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43672 deadline: 1732093227907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:27,924 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:27,924 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:27,924 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:27,926 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:27,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43628 deadline: 1732093227907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:27,928 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:27,928 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:27,928 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:27,929 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:27,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43608 deadline: 1732093227919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:27,930 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:27,930 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:27,930 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:27,937 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/C/86e23a1f4fac4b60baca926bfe626ad0 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/C/86e23a1f4fac4b60baca926bfe626ad0 2024-11-20T08:59:27,957 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1c52b94a9e6adce80d2d034c31b67cfb/C of 1c52b94a9e6adce80d2d034c31b67cfb into 86e23a1f4fac4b60baca926bfe626ad0(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T08:59:27,957 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1c52b94a9e6adce80d2d034c31b67cfb: 2024-11-20T08:59:27,957 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., storeName=1c52b94a9e6adce80d2d034c31b67cfb/C, priority=12, startTime=1732093167687; duration=0sec 2024-11-20T08:59:27,957 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T08:59:27,958 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1c52b94a9e6adce80d2d034c31b67cfb:C 2024-11-20T08:59:27,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-20T08:59:27,978 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T08:59:27,978 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T08:59:27,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees 2024-11-20T08:59:27,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-20T08:59:27,985 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T08:59:27,986 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T08:59:27,986 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T08:59:28,023 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:28,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43644 deadline: 1732093228018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:28,024 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:28,024 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:28,024 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:28,028 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:28,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43620 deadline: 1732093228027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:28,029 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:28,029 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:28,030 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:28,030 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:28,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43672 deadline: 1732093228027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:28,030 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:28,031 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:28,031 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:28,041 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:28,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43628 deadline: 1732093228038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:28,042 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:28,042 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:28,043 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:28,043 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:28,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43608 deadline: 1732093228038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:28,044 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:28,044 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:28,044 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:28,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-20T08:59:28,141 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-20T08:59:28,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb. 2024-11-20T08:59:28,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb. as already flushing 2024-11-20T08:59:28,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb. 2024-11-20T08:59:28,142 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 1c52b94a9e6adce80d2d034c31b67cfb, NAME => 'TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T08:59:28,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 1c52b94a9e6adce80d2d034c31b67cfb, NAME => 'TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T08:59:28,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1c52b94a9e6adce80d2d034c31b67cfb, NAME => 'TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1c52b94a9e6adce80d2d034c31b67cfb, NAME => 'TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T08:59:28,227 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/A/8a99370765c549529790491b7129bc4b as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/8a99370765c549529790491b7129bc4b 2024-11-20T08:59:28,228 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=89 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/A/54173cb23d0b4a6a80f35a7c576c08e3 2024-11-20T08:59:28,232 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:28,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43644 deadline: 1732093228228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:28,233 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:28,234 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:28,234 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:28,251 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/B/ccede917c6fb47bf82297b2c25647b36 is 50, key is test_row_0/B:col10/1732093167759/Put/seqid=0 2024-11-20T08:59:28,251 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1c52b94a9e6adce80d2d034c31b67cfb/A of 1c52b94a9e6adce80d2d034c31b67cfb into 8a99370765c549529790491b7129bc4b(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T08:59:28,251 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1c52b94a9e6adce80d2d034c31b67cfb: 2024-11-20T08:59:28,252 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., storeName=1c52b94a9e6adce80d2d034c31b67cfb/A, priority=12, startTime=1732093167664; duration=0sec 2024-11-20T08:59:28,252 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T08:59:28,252 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1c52b94a9e6adce80d2d034c31b67cfb:A 2024-11-20T08:59:28,252 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:28,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43620 deadline: 1732093228237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:28,253 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:28,253 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:28,254 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:28,254 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:28,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43672 deadline: 1732093228238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:28,254 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:28,255 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:28,255 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:28,255 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:28,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43608 deadline: 1732093228247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:28,256 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:28,256 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:28,257 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:28,256 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:28,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43628 deadline: 1732093228247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:28,257 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:28,257 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:28,258 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:28,281 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073741853_1029 (size=9657) 2024-11-20T08:59:28,282 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=89 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/B/ccede917c6fb47bf82297b2c25647b36 2024-11-20T08:59:28,297 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-20T08:59:28,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-20T08:59:28,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb. 2024-11-20T08:59:28,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb. as already flushing 2024-11-20T08:59:28,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb. 2024-11-20T08:59:28,298 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 1c52b94a9e6adce80d2d034c31b67cfb, NAME => 'TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T08:59:28,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 1c52b94a9e6adce80d2d034c31b67cfb, NAME => 'TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T08:59:28,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1c52b94a9e6adce80d2d034c31b67cfb, NAME => 'TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1c52b94a9e6adce80d2d034c31b67cfb, NAME => 'TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T08:59:28,303 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/C/7d1751fdc5ff44bc935c8b93c1d3bb07 is 50, key is test_row_0/C:col10/1732093167759/Put/seqid=0 2024-11-20T08:59:28,333 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073741854_1030 (size=9657) 2024-11-20T08:59:28,454 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-20T08:59:28,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb. 2024-11-20T08:59:28,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb. as already flushing 2024-11-20T08:59:28,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb. 2024-11-20T08:59:28,455 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 1c52b94a9e6adce80d2d034c31b67cfb, NAME => 'TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T08:59:28,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 1c52b94a9e6adce80d2d034c31b67cfb, NAME => 'TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T08:59:28,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1c52b94a9e6adce80d2d034c31b67cfb, NAME => 'TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1c52b94a9e6adce80d2d034c31b67cfb, NAME => 'TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T08:59:28,540 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:28,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43644 deadline: 1732093228538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:28,541 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:28,541 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:28,541 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:28,569 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:28,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43620 deadline: 1732093228567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:28,571 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:28,571 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:28,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43672 deadline: 1732093228568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:28,571 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:28,571 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:28,571 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:28,572 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:28,572 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:28,572 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:28,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43628 deadline: 1732093228568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:28,573 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:28,573 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:28,573 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:28,573 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:28,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43608 deadline: 1732093228569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:28,573 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:28,574 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:28,574 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:28,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-20T08:59:28,610 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-20T08:59:28,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb. 2024-11-20T08:59:28,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb. as already flushing 2024-11-20T08:59:28,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb. 2024-11-20T08:59:28,610 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 1c52b94a9e6adce80d2d034c31b67cfb, NAME => 'TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T08:59:28,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 1c52b94a9e6adce80d2d034c31b67cfb, NAME => 'TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T08:59:28,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1c52b94a9e6adce80d2d034c31b67cfb, NAME => 'TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1c52b94a9e6adce80d2d034c31b67cfb, NAME => 'TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T08:59:28,637 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-20T08:59:28,736 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=89 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/C/7d1751fdc5ff44bc935c8b93c1d3bb07 2024-11-20T08:59:28,736 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-20T08:59:28,760 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/A/54173cb23d0b4a6a80f35a7c576c08e3 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/54173cb23d0b4a6a80f35a7c576c08e3 2024-11-20T08:59:28,765 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-20T08:59:28,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb. 2024-11-20T08:59:28,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb. as already flushing 2024-11-20T08:59:28,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb. 2024-11-20T08:59:28,766 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 1c52b94a9e6adce80d2d034c31b67cfb, NAME => 'TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T08:59:28,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 1c52b94a9e6adce80d2d034c31b67cfb, NAME => 'TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T08:59:28,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1c52b94a9e6adce80d2d034c31b67cfb, NAME => 'TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1c52b94a9e6adce80d2d034c31b67cfb, NAME => 'TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T08:59:28,777 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/54173cb23d0b4a6a80f35a7c576c08e3, entries=150, sequenceid=89, filesize=11.7 K 2024-11-20T08:59:28,779 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/B/ccede917c6fb47bf82297b2c25647b36 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/ccede917c6fb47bf82297b2c25647b36 2024-11-20T08:59:28,792 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/ccede917c6fb47bf82297b2c25647b36, entries=100, sequenceid=89, filesize=9.4 K 2024-11-20T08:59:28,797 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/C/7d1751fdc5ff44bc935c8b93c1d3bb07 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/C/7d1751fdc5ff44bc935c8b93c1d3bb07 2024-11-20T08:59:28,814 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/C/7d1751fdc5ff44bc935c8b93c1d3bb07, entries=100, sequenceid=89, filesize=9.4 K 2024-11-20T08:59:28,816 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=161.02 KB/164880 for 1c52b94a9e6adce80d2d034c31b67cfb in 1043ms, sequenceid=89, compaction requested=false 2024-11-20T08:59:28,817 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1c52b94a9e6adce80d2d034c31b67cfb: 2024-11-20T08:59:28,922 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-20T08:59:28,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb. 2024-11-20T08:59:28,923 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing 1c52b94a9e6adce80d2d034c31b67cfb 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-20T08:59:28,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/A/b55dc0d1aa924a2491365bda80d65b82 is 50, key is test_row_0/A:col10/1732093167897/Put/seqid=0 2024-11-20T08:59:28,957 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073741855_1031 (size=12001) 2024-11-20T08:59:28,962 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/A/b55dc0d1aa924a2491365bda80d65b82 2024-11-20T08:59:28,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/B/ab247e55e1464dadb2ce6dcc7bbba93e is 50, key is test_row_0/B:col10/1732093167897/Put/seqid=0 2024-11-20T08:59:28,997 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073741856_1032 (size=12001) 2024-11-20T08:59:28,999 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/B/ab247e55e1464dadb2ce6dcc7bbba93e 2024-11-20T08:59:29,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/C/e496a80c051049a08997ed69b0103949 is 50, key is test_row_0/C:col10/1732093167897/Put/seqid=0 2024-11-20T08:59:29,051 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb. as already flushing 2024-11-20T08:59:29,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(8855): Flush requested on 1c52b94a9e6adce80d2d034c31b67cfb 2024-11-20T08:59:29,081 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073741857_1033 (size=12001) 2024-11-20T08:59:29,083 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/C/e496a80c051049a08997ed69b0103949 2024-11-20T08:59:29,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/A/b55dc0d1aa924a2491365bda80d65b82 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/b55dc0d1aa924a2491365bda80d65b82 2024-11-20T08:59:29,101 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:29,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43672 deadline: 1732093229086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:29,102 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:29,102 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:29,102 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:29,102 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:29,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43620 deadline: 1732093229086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:29,103 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:29,103 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:29,103 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:29,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43628 deadline: 1732093229089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:29,103 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:29,103 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:29,104 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:29,104 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:29,104 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:29,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43608 deadline: 1732093229087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:29,104 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:29,104 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:29,105 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:29,107 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:29,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43644 deadline: 1732093229094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:29,108 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:29,108 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:29,109 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:29,117 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/b55dc0d1aa924a2491365bda80d65b82, entries=150, sequenceid=119, filesize=11.7 K 2024-11-20T08:59:29,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-20T08:59:29,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/B/ab247e55e1464dadb2ce6dcc7bbba93e as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/ab247e55e1464dadb2ce6dcc7bbba93e 2024-11-20T08:59:29,131 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/ab247e55e1464dadb2ce6dcc7bbba93e, entries=150, sequenceid=119, filesize=11.7 K 2024-11-20T08:59:29,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/C/e496a80c051049a08997ed69b0103949 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/C/e496a80c051049a08997ed69b0103949 2024-11-20T08:59:29,150 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/C/e496a80c051049a08997ed69b0103949, entries=150, sequenceid=119, filesize=11.7 K 2024-11-20T08:59:29,153 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=46.96 KB/48090 for 1c52b94a9e6adce80d2d034c31b67cfb in 229ms, sequenceid=119, compaction requested=true 2024-11-20T08:59:29,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for 1c52b94a9e6adce80d2d034c31b67cfb: 2024-11-20T08:59:29,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb. 2024-11-20T08:59:29,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-11-20T08:59:29,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-11-20T08:59:29,165 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-11-20T08:59:29,165 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1700 sec 2024-11-20T08:59:29,172 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees in 1.1900 sec 2024-11-20T08:59:29,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(8855): Flush requested on 1c52b94a9e6adce80d2d034c31b67cfb 2024-11-20T08:59:29,212 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T08:59:29,213 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1c52b94a9e6adce80d2d034c31b67cfb 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-20T08:59:29,236 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/A/30d99cb2d5a8402887d431d2b8c511f8 is 50, key is test_row_0/A:col10/1732093169209/Put/seqid=0 2024-11-20T08:59:29,246 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073741858_1034 (size=12051) 2024-11-20T08:59:29,247 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/A/30d99cb2d5a8402887d431d2b8c511f8 2024-11-20T08:59:29,262 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/B/9cbb9795511a463281f5d634616bcbf0 is 50, key is test_row_0/B:col10/1732093169209/Put/seqid=0 2024-11-20T08:59:29,276 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073741859_1035 (size=12051) 2024-11-20T08:59:29,278 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/B/9cbb9795511a463281f5d634616bcbf0 2024-11-20T08:59:29,296 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/C/937e0010b140494893f21df2e3750358 is 50, key is test_row_0/C:col10/1732093169209/Put/seqid=0 2024-11-20T08:59:29,323 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073741860_1036 (size=12051) 2024-11-20T08:59:29,332 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/C/937e0010b140494893f21df2e3750358 2024-11-20T08:59:29,349 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/A/30d99cb2d5a8402887d431d2b8c511f8 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/30d99cb2d5a8402887d431d2b8c511f8 2024-11-20T08:59:29,352 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:4773) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:4699) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.mutate(HRegion.java:5194) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.mutate(HRegion.java:5188) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.mutate(HRegion.java:5184) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3333) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:29,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43672 deadline: 1732093229345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:29,353 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:4773) at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:4699) at org.apache.hadoop.hbase.regionserver.HRegion.mutate(HRegion.java:5194) at org.apache.hadoop.hbase.regionserver.HRegion.mutate(HRegion.java:5188) at org.apache.hadoop.hbase.regionserver.HRegion.mutate(HRegion.java:5184) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3333) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:29,354 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:4773) at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:4699) at org.apache.hadoop.hbase.regionserver.HRegion.mutate(HRegion.java:5194) at org.apache.hadoop.hbase.regionserver.HRegion.mutate(HRegion.java:5188) at org.apache.hadoop.hbase.regionserver.HRegion.mutate(HRegion.java:5184) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3333) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:29,354 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:29,355 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:29,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43620 deadline: 1732093229346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:29,355 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:29,355 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:29,356 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:29,359 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:29,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43644 deadline: 1732093229349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:29,360 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:29,360 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:29,360 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:29,361 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/30d99cb2d5a8402887d431d2b8c511f8, entries=150, sequenceid=131, filesize=11.8 K 2024-11-20T08:59:29,362 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:29,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43628 deadline: 1732093229354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:29,363 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:29,363 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:29,363 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:29,363 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/B/9cbb9795511a463281f5d634616bcbf0 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/9cbb9795511a463281f5d634616bcbf0 2024-11-20T08:59:29,373 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/9cbb9795511a463281f5d634616bcbf0, entries=150, sequenceid=131, filesize=11.8 K 2024-11-20T08:59:29,375 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/C/937e0010b140494893f21df2e3750358 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/C/937e0010b140494893f21df2e3750358 2024-11-20T08:59:29,386 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/C/937e0010b140494893f21df2e3750358, entries=150, sequenceid=131, filesize=11.8 K 2024-11-20T08:59:29,390 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 1c52b94a9e6adce80d2d034c31b67cfb in 176ms, sequenceid=131, compaction requested=true 2024-11-20T08:59:29,390 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1c52b94a9e6adce80d2d034c31b67cfb: 2024-11-20T08:59:29,390 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T08:59:29,391 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1c52b94a9e6adce80d2d034c31b67cfb:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T08:59:29,391 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T08:59:29,391 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T08:59:29,391 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1c52b94a9e6adce80d2d034c31b67cfb:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T08:59:29,391 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T08:59:29,391 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1c52b94a9e6adce80d2d034c31b67cfb:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T08:59:29,391 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T08:59:29,394 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48188 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T08:59:29,394 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HStore(1541): 1c52b94a9e6adce80d2d034c31b67cfb/A is initiating minor compaction (all files) 2024-11-20T08:59:29,394 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1c52b94a9e6adce80d2d034c31b67cfb/A in TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb. 2024-11-20T08:59:29,394 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/8a99370765c549529790491b7129bc4b, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/54173cb23d0b4a6a80f35a7c576c08e3, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/b55dc0d1aa924a2491365bda80d65b82, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/30d99cb2d5a8402887d431d2b8c511f8] into tmpdir=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp, totalSize=47.1 K 2024-11-20T08:59:29,395 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 45848 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T08:59:29,395 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1541): 1c52b94a9e6adce80d2d034c31b67cfb/B is initiating minor compaction (all files) 2024-11-20T08:59:29,395 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1c52b94a9e6adce80d2d034c31b67cfb/B in TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb. 2024-11-20T08:59:29,395 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/a0f5cdf99a1841e6bb7c9d5933220ad1, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/ccede917c6fb47bf82297b2c25647b36, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/ab247e55e1464dadb2ce6dcc7bbba93e, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/9cbb9795511a463281f5d634616bcbf0] into tmpdir=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp, totalSize=44.8 K 2024-11-20T08:59:29,396 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8a99370765c549529790491b7129bc4b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1732093167411 2024-11-20T08:59:29,396 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting a0f5cdf99a1841e6bb7c9d5933220ad1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1732093167411 2024-11-20T08:59:29,398 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting ccede917c6fb47bf82297b2c25647b36, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1732093167759 2024-11-20T08:59:29,397 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.Compactor(225): Compacting 54173cb23d0b4a6a80f35a7c576c08e3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1732093167750 2024-11-20T08:59:29,400 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting ab247e55e1464dadb2ce6dcc7bbba93e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1732093167890 2024-11-20T08:59:29,401 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.Compactor(225): Compacting b55dc0d1aa924a2491365bda80d65b82, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1732093167890 2024-11-20T08:59:29,402 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting 9cbb9795511a463281f5d634616bcbf0, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1732093169085 2024-11-20T08:59:29,402 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.Compactor(225): Compacting 30d99cb2d5a8402887d431d2b8c511f8, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1732093169085 2024-11-20T08:59:29,435 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1c52b94a9e6adce80d2d034c31b67cfb#A#compaction#24 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T08:59:29,437 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/A/6df65a221ea64367a50dbf9cb107dd8a is 50, key is test_row_0/A:col10/1732093169209/Put/seqid=0 2024-11-20T08:59:29,447 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1c52b94a9e6adce80d2d034c31b67cfb#B#compaction#25 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T08:59:29,448 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/B/9003c5e71a2e48558bb3a3053d91953b is 50, key is test_row_0/B:col10/1732093169209/Put/seqid=0 2024-11-20T08:59:29,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(8855): Flush requested on 1c52b94a9e6adce80d2d034c31b67cfb 2024-11-20T08:59:29,462 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T08:59:29,462 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1c52b94a9e6adce80d2d034c31b67cfb 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-20T08:59:29,481 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/A/f947288e99de4974a55d6d8c54a627dc is 50, key is test_row_0/A:col10/1732093169345/Put/seqid=0 2024-11-20T08:59:29,496 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073741862_1038 (size=12325) 2024-11-20T08:59:29,498 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073741861_1037 (size=12325) 2024-11-20T08:59:29,501 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:29,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43672 deadline: 1732093229491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:29,502 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:29,502 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:29,502 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:29,506 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:29,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43628 deadline: 1732093229494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:29,509 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:29,509 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:29,509 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:29,509 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:29,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43620 deadline: 1732093229499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:29,510 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:29,510 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:29,510 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:29,510 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:29,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43644 deadline: 1732093229501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:29,511 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:29,511 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:29,511 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:29,515 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073741863_1039 (size=12151) 2024-11-20T08:59:29,515 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/A/6df65a221ea64367a50dbf9cb107dd8a as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/6df65a221ea64367a50dbf9cb107dd8a 2024-11-20T08:59:29,517 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/A/f947288e99de4974a55d6d8c54a627dc 2024-11-20T08:59:29,536 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1c52b94a9e6adce80d2d034c31b67cfb/A of 1c52b94a9e6adce80d2d034c31b67cfb into 6df65a221ea64367a50dbf9cb107dd8a(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T08:59:29,536 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1c52b94a9e6adce80d2d034c31b67cfb: 2024-11-20T08:59:29,537 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., storeName=1c52b94a9e6adce80d2d034c31b67cfb/A, priority=12, startTime=1732093169390; duration=0sec 2024-11-20T08:59:29,537 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T08:59:29,537 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1c52b94a9e6adce80d2d034c31b67cfb:A 2024-11-20T08:59:29,537 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T08:59:29,541 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 45848 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T08:59:29,541 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HStore(1541): 1c52b94a9e6adce80d2d034c31b67cfb/C is initiating minor compaction (all files) 2024-11-20T08:59:29,541 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1c52b94a9e6adce80d2d034c31b67cfb/C in TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb. 2024-11-20T08:59:29,542 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/C/86e23a1f4fac4b60baca926bfe626ad0, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/C/7d1751fdc5ff44bc935c8b93c1d3bb07, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/C/e496a80c051049a08997ed69b0103949, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/C/937e0010b140494893f21df2e3750358] into tmpdir=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp, totalSize=44.8 K 2024-11-20T08:59:29,543 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.Compactor(225): Compacting 86e23a1f4fac4b60baca926bfe626ad0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1732093167411 2024-11-20T08:59:29,543 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7d1751fdc5ff44bc935c8b93c1d3bb07, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1732093167759 2024-11-20T08:59:29,544 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.Compactor(225): Compacting e496a80c051049a08997ed69b0103949, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1732093167890 2024-11-20T08:59:29,545 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.Compactor(225): Compacting 937e0010b140494893f21df2e3750358, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1732093169085 2024-11-20T08:59:29,547 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/B/81e2c807fc6b4bb892360f41155348a1 is 50, key is test_row_0/B:col10/1732093169345/Put/seqid=0 2024-11-20T08:59:29,567 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1c52b94a9e6adce80d2d034c31b67cfb#C#compaction#28 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T08:59:29,568 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/C/804fb3a729c04009a44cda091b94774f is 50, key is test_row_0/C:col10/1732093169209/Put/seqid=0 2024-11-20T08:59:29,583 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073741864_1040 (size=12151) 2024-11-20T08:59:29,584 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/B/81e2c807fc6b4bb892360f41155348a1 2024-11-20T08:59:29,599 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073741865_1041 (size=12325) 2024-11-20T08:59:29,611 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/C/b38ab494a2cf4f8faf649c46a8a08272 is 50, key is test_row_0/C:col10/1732093169345/Put/seqid=0 2024-11-20T08:59:29,614 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:29,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43672 deadline: 1732093229608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:29,615 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:29,615 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:29,615 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:29,617 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/C/804fb3a729c04009a44cda091b94774f as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/C/804fb3a729c04009a44cda091b94774f 2024-11-20T08:59:29,621 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:29,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43644 deadline: 1732093229618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:29,622 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:29,622 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:29,622 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:29,622 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:29,622 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:29,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43628 deadline: 1732093229620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:29,623 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:29,623 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:29,623 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:29,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43620 deadline: 1732093229620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:29,625 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:29,625 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:29,625 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:29,631 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1c52b94a9e6adce80d2d034c31b67cfb/C of 1c52b94a9e6adce80d2d034c31b67cfb into 804fb3a729c04009a44cda091b94774f(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T08:59:29,632 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1c52b94a9e6adce80d2d034c31b67cfb: 2024-11-20T08:59:29,632 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., storeName=1c52b94a9e6adce80d2d034c31b67cfb/C, priority=12, startTime=1732093169391; duration=0sec 2024-11-20T08:59:29,632 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073741866_1042 (size=12151) 2024-11-20T08:59:29,633 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T08:59:29,633 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1c52b94a9e6adce80d2d034c31b67cfb:C 2024-11-20T08:59:29,633 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/C/b38ab494a2cf4f8faf649c46a8a08272 2024-11-20T08:59:29,646 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-20T08:59:29,647 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-20T08:59:29,649 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-20T08:59:29,649 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-20T08:59:29,649 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-20T08:59:29,649 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-20T08:59:29,651 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-20T08:59:29,651 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-20T08:59:29,653 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/A/f947288e99de4974a55d6d8c54a627dc as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/f947288e99de4974a55d6d8c54a627dc 2024-11-20T08:59:29,667 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/f947288e99de4974a55d6d8c54a627dc, entries=150, sequenceid=156, filesize=11.9 K 2024-11-20T08:59:29,669 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/B/81e2c807fc6b4bb892360f41155348a1 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/81e2c807fc6b4bb892360f41155348a1 2024-11-20T08:59:29,678 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/81e2c807fc6b4bb892360f41155348a1, entries=150, sequenceid=156, filesize=11.9 K 2024-11-20T08:59:29,680 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/C/b38ab494a2cf4f8faf649c46a8a08272 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/C/b38ab494a2cf4f8faf649c46a8a08272 2024-11-20T08:59:29,693 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/C/b38ab494a2cf4f8faf649c46a8a08272, entries=150, sequenceid=156, filesize=11.9 K 2024-11-20T08:59:29,695 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 1c52b94a9e6adce80d2d034c31b67cfb in 233ms, sequenceid=156, compaction requested=false 2024-11-20T08:59:29,695 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1c52b94a9e6adce80d2d034c31b67cfb: 2024-11-20T08:59:29,831 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T08:59:29,831 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1c52b94a9e6adce80d2d034c31b67cfb 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-20T08:59:29,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(8855): Flush requested on 1c52b94a9e6adce80d2d034c31b67cfb 2024-11-20T08:59:29,849 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/A/20cecef2ca6c43c184c816a58d96b6ba is 50, key is test_row_0/A:col10/1732093169497/Put/seqid=0 2024-11-20T08:59:29,882 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073741867_1043 (size=12151) 2024-11-20T08:59:29,883 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=171 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/A/20cecef2ca6c43c184c816a58d96b6ba 2024-11-20T08:59:29,891 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:29,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43672 deadline: 1732093229885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:29,892 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:29,893 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:29,893 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:29,896 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:29,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43628 deadline: 1732093229889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:29,897 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:29,897 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:29,897 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:29,897 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:29,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43620 deadline: 1732093229890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:29,898 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:29,898 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:29,898 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:29,905 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/B/ac6605a3d8ca4d5ea9bb8abbbadad5d1 is 50, key is test_row_0/B:col10/1732093169497/Put/seqid=0 2024-11-20T08:59:29,907 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:29,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43644 deadline: 1732093229897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:29,908 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:29,908 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:29,908 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:29,916 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/B/9003c5e71a2e48558bb3a3053d91953b as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/9003c5e71a2e48558bb3a3053d91953b 2024-11-20T08:59:29,929 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1c52b94a9e6adce80d2d034c31b67cfb/B of 1c52b94a9e6adce80d2d034c31b67cfb into 9003c5e71a2e48558bb3a3053d91953b(size=12.0 K), total size for store is 23.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T08:59:29,929 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1c52b94a9e6adce80d2d034c31b67cfb: 2024-11-20T08:59:29,929 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., storeName=1c52b94a9e6adce80d2d034c31b67cfb/B, priority=12, startTime=1732093169391; duration=0sec 2024-11-20T08:59:29,929 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T08:59:29,929 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1c52b94a9e6adce80d2d034c31b67cfb:B 2024-11-20T08:59:29,959 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073741868_1044 (size=12151) 2024-11-20T08:59:30,000 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:30,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43672 deadline: 1732093229997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:30,001 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:30,001 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:30,001 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:30,012 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:30,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43628 deadline: 1732093230007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:30,013 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:30,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43620 deadline: 1732093230007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:30,013 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:30,013 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:30,013 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:30,015 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:30,015 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:30,015 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:30,018 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:30,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43644 deadline: 1732093230018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:30,018 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:30,019 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:30,019 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:30,117 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:30,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43608 deadline: 1732093230117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:30,122 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to be1eb2620e0e,34223,1732093159960 for 'test_row_1' in 1c52b94a9e6adce80d2d034c31b67cfb of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2200 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-20T08:59:30,124 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:30,125 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:30,125 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:30,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-20T08:59:30,127 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T08:59:30,128 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T08:59:30,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees 2024-11-20T08:59:30,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-20T08:59:30,133 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T08:59:30,135 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T08:59:30,135 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T08:59:30,208 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:30,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43672 deadline: 1732093230207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:30,209 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:30,209 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:30,209 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:30,220 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:30,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43628 deadline: 1732093230218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:30,221 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:30,221 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:30,221 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:30,232 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:30,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43620 deadline: 1732093230227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:30,233 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:30,233 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:30,233 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:30,233 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:30,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43644 deadline: 1732093230227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:30,234 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:30,234 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:30,234 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:30,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-20T08:59:30,290 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-20T08:59:30,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb. 2024-11-20T08:59:30,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb. as already flushing 2024-11-20T08:59:30,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb. 2024-11-20T08:59:30,291 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 1c52b94a9e6adce80d2d034c31b67cfb, NAME => 'TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T08:59:30,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 1c52b94a9e6adce80d2d034c31b67cfb, NAME => 'TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T08:59:30,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1c52b94a9e6adce80d2d034c31b67cfb, NAME => 'TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1c52b94a9e6adce80d2d034c31b67cfb, NAME => 'TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T08:59:30,363 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=171 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/B/ac6605a3d8ca4d5ea9bb8abbbadad5d1 2024-11-20T08:59:30,378 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/C/b3e41d70621844ba8f3d2ad66c53a171 is 50, key is test_row_0/C:col10/1732093169497/Put/seqid=0 2024-11-20T08:59:30,395 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073741869_1045 (size=12151) 2024-11-20T08:59:30,397 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=171 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/C/b3e41d70621844ba8f3d2ad66c53a171 2024-11-20T08:59:30,411 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/A/20cecef2ca6c43c184c816a58d96b6ba as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/20cecef2ca6c43c184c816a58d96b6ba 2024-11-20T08:59:30,423 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/20cecef2ca6c43c184c816a58d96b6ba, entries=150, sequenceid=171, filesize=11.9 K 2024-11-20T08:59:30,425 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/B/ac6605a3d8ca4d5ea9bb8abbbadad5d1 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/ac6605a3d8ca4d5ea9bb8abbbadad5d1 2024-11-20T08:59:30,434 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/ac6605a3d8ca4d5ea9bb8abbbadad5d1, entries=150, sequenceid=171, filesize=11.9 K 2024-11-20T08:59:30,436 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/C/b3e41d70621844ba8f3d2ad66c53a171 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/C/b3e41d70621844ba8f3d2ad66c53a171 2024-11-20T08:59:30,446 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-20T08:59:30,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb. 2024-11-20T08:59:30,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb. as already flushing 2024-11-20T08:59:30,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-20T08:59:30,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb. 2024-11-20T08:59:30,447 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 1c52b94a9e6adce80d2d034c31b67cfb, NAME => 'TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T08:59:30,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 1c52b94a9e6adce80d2d034c31b67cfb, NAME => 'TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T08:59:30,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1c52b94a9e6adce80d2d034c31b67cfb, NAME => 'TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1c52b94a9e6adce80d2d034c31b67cfb, NAME => 'TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T08:59:30,450 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/C/b3e41d70621844ba8f3d2ad66c53a171, entries=150, sequenceid=171, filesize=11.9 K 2024-11-20T08:59:30,452 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=140.89 KB/144270 for 1c52b94a9e6adce80d2d034c31b67cfb in 621ms, sequenceid=171, compaction requested=true 2024-11-20T08:59:30,452 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1c52b94a9e6adce80d2d034c31b67cfb: 2024-11-20T08:59:30,452 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1c52b94a9e6adce80d2d034c31b67cfb:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T08:59:30,452 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T08:59:30,452 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T08:59:30,452 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1c52b94a9e6adce80d2d034c31b67cfb:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T08:59:30,453 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T08:59:30,453 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T08:59:30,453 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1c52b94a9e6adce80d2d034c31b67cfb:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T08:59:30,453 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T08:59:30,454 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36627 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T08:59:30,454 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1541): 1c52b94a9e6adce80d2d034c31b67cfb/B is initiating minor compaction (all files) 2024-11-20T08:59:30,454 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1c52b94a9e6adce80d2d034c31b67cfb/B in TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb. 2024-11-20T08:59:30,455 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/9003c5e71a2e48558bb3a3053d91953b, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/81e2c807fc6b4bb892360f41155348a1, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/ac6605a3d8ca4d5ea9bb8abbbadad5d1] into tmpdir=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp, totalSize=35.8 K 2024-11-20T08:59:30,455 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36627 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T08:59:30,455 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HStore(1541): 1c52b94a9e6adce80d2d034c31b67cfb/A is initiating minor compaction (all files) 2024-11-20T08:59:30,455 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1c52b94a9e6adce80d2d034c31b67cfb/A in TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb. 2024-11-20T08:59:30,455 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting 9003c5e71a2e48558bb3a3053d91953b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1732093169085 2024-11-20T08:59:30,455 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/6df65a221ea64367a50dbf9cb107dd8a, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/f947288e99de4974a55d6d8c54a627dc, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/20cecef2ca6c43c184c816a58d96b6ba] into tmpdir=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp, totalSize=35.8 K 2024-11-20T08:59:30,456 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6df65a221ea64367a50dbf9cb107dd8a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1732093169085 2024-11-20T08:59:30,456 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting 81e2c807fc6b4bb892360f41155348a1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1732093169342 2024-11-20T08:59:30,458 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.Compactor(225): Compacting f947288e99de4974a55d6d8c54a627dc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1732093169342 2024-11-20T08:59:30,458 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting ac6605a3d8ca4d5ea9bb8abbbadad5d1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1732093169497 2024-11-20T08:59:30,459 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.Compactor(225): Compacting 20cecef2ca6c43c184c816a58d96b6ba, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1732093169497 2024-11-20T08:59:30,474 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1c52b94a9e6adce80d2d034c31b67cfb#B#compaction#33 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T08:59:30,474 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/B/9799a5642bfe40259b188a37202879cb is 50, key is test_row_0/B:col10/1732093169497/Put/seqid=0 2024-11-20T08:59:30,482 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1c52b94a9e6adce80d2d034c31b67cfb#A#compaction#34 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T08:59:30,483 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/A/f8265531d4f14044b8a1ad0cac34e7fe is 50, key is test_row_0/A:col10/1732093169497/Put/seqid=0 2024-11-20T08:59:30,498 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073741871_1047 (size=12527) 2024-11-20T08:59:30,499 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073741870_1046 (size=12527) 2024-11-20T08:59:30,517 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/B/9799a5642bfe40259b188a37202879cb as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/9799a5642bfe40259b188a37202879cb 2024-11-20T08:59:30,517 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/A/f8265531d4f14044b8a1ad0cac34e7fe as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/f8265531d4f14044b8a1ad0cac34e7fe 2024-11-20T08:59:30,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(8855): Flush requested on 1c52b94a9e6adce80d2d034c31b67cfb 2024-11-20T08:59:30,526 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T08:59:30,527 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1c52b94a9e6adce80d2d034c31b67cfb 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-20T08:59:30,533 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1c52b94a9e6adce80d2d034c31b67cfb/B of 1c52b94a9e6adce80d2d034c31b67cfb into 9799a5642bfe40259b188a37202879cb(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T08:59:30,533 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1c52b94a9e6adce80d2d034c31b67cfb: 2024-11-20T08:59:30,533 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., storeName=1c52b94a9e6adce80d2d034c31b67cfb/B, priority=13, startTime=1732093170452; duration=0sec 2024-11-20T08:59:30,533 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T08:59:30,534 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1c52b94a9e6adce80d2d034c31b67cfb:B 2024-11-20T08:59:30,534 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T08:59:30,537 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1c52b94a9e6adce80d2d034c31b67cfb/A of 1c52b94a9e6adce80d2d034c31b67cfb into f8265531d4f14044b8a1ad0cac34e7fe(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T08:59:30,537 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1c52b94a9e6adce80d2d034c31b67cfb: 2024-11-20T08:59:30,537 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., storeName=1c52b94a9e6adce80d2d034c31b67cfb/A, priority=13, startTime=1732093170452; duration=0sec 2024-11-20T08:59:30,537 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T08:59:30,537 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1c52b94a9e6adce80d2d034c31b67cfb:A 2024-11-20T08:59:30,542 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36627 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T08:59:30,542 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1541): 1c52b94a9e6adce80d2d034c31b67cfb/C is initiating minor compaction (all files) 2024-11-20T08:59:30,542 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1c52b94a9e6adce80d2d034c31b67cfb/C in TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb. 2024-11-20T08:59:30,542 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/C/804fb3a729c04009a44cda091b94774f, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/C/b38ab494a2cf4f8faf649c46a8a08272, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/C/b3e41d70621844ba8f3d2ad66c53a171] into tmpdir=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp, totalSize=35.8 K 2024-11-20T08:59:30,543 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting 804fb3a729c04009a44cda091b94774f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1732093169085 2024-11-20T08:59:30,544 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting b38ab494a2cf4f8faf649c46a8a08272, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1732093169342 2024-11-20T08:59:30,545 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting b3e41d70621844ba8f3d2ad66c53a171, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1732093169497 2024-11-20T08:59:30,550 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/A/b37f571b76584fbb8c4b6a159dad0bc8 is 50, key is test_row_0/A:col10/1732093169888/Put/seqid=0 2024-11-20T08:59:30,556 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:30,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43620 deadline: 1732093230546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:30,557 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:30,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43644 deadline: 1732093230548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:30,557 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:30,557 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:30,557 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:30,557 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:30,558 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:30,558 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:30,560 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:30,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43628 deadline: 1732093230552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:30,561 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:30,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43672 deadline: 1732093230555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:30,561 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:30,561 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:30,561 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:30,561 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:30,562 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:30,562 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:30,579 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1c52b94a9e6adce80d2d034c31b67cfb#C#compaction#36 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T08:59:30,580 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/C/9f5278a2f6b3484ea4ed75021ab2baf8 is 50, key is test_row_0/C:col10/1732093169497/Put/seqid=0 2024-11-20T08:59:30,597 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073741872_1048 (size=12151) 2024-11-20T08:59:30,599 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=197 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/A/b37f571b76584fbb8c4b6a159dad0bc8 2024-11-20T08:59:30,601 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-20T08:59:30,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb. 2024-11-20T08:59:30,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb. as already flushing 2024-11-20T08:59:30,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb. 2024-11-20T08:59:30,601 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 1c52b94a9e6adce80d2d034c31b67cfb, NAME => 'TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T08:59:30,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 1c52b94a9e6adce80d2d034c31b67cfb, NAME => 'TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T08:59:30,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1c52b94a9e6adce80d2d034c31b67cfb, NAME => 'TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1c52b94a9e6adce80d2d034c31b67cfb, NAME => 'TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T08:59:30,629 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073741873_1049 (size=12527) 2024-11-20T08:59:30,635 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/B/d47461f002a442bba66ba2b1f5e06318 is 50, key is test_row_0/B:col10/1732093169888/Put/seqid=0 2024-11-20T08:59:30,645 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073741874_1050 (size=12151) 2024-11-20T08:59:30,647 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=197 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/B/d47461f002a442bba66ba2b1f5e06318 2024-11-20T08:59:30,661 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/C/cf14c4a783214851a7a97272dc5bc134 is 50, key is test_row_0/C:col10/1732093169888/Put/seqid=0 2024-11-20T08:59:30,669 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:30,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43644 deadline: 1732093230667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:30,670 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:30,670 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:30,670 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:30,671 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:30,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43672 deadline: 1732093230667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:30,672 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:30,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43628 deadline: 1732093230667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:30,672 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:30,672 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:30,672 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:30,673 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:30,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43620 deadline: 1732093230667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:30,673 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:30,673 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:30,673 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:30,673 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:30,674 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:30,674 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:30,694 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073741875_1051 (size=12151) 2024-11-20T08:59:30,695 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=197 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/C/cf14c4a783214851a7a97272dc5bc134 2024-11-20T08:59:30,706 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/A/b37f571b76584fbb8c4b6a159dad0bc8 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/b37f571b76584fbb8c4b6a159dad0bc8 2024-11-20T08:59:30,720 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/b37f571b76584fbb8c4b6a159dad0bc8, entries=150, sequenceid=197, filesize=11.9 K 2024-11-20T08:59:30,723 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/B/d47461f002a442bba66ba2b1f5e06318 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/d47461f002a442bba66ba2b1f5e06318 2024-11-20T08:59:30,746 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/d47461f002a442bba66ba2b1f5e06318, entries=150, sequenceid=197, filesize=11.9 K 2024-11-20T08:59:30,751 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/C/cf14c4a783214851a7a97272dc5bc134 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/C/cf14c4a783214851a7a97272dc5bc134 2024-11-20T08:59:30,756 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-20T08:59:30,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb. 2024-11-20T08:59:30,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb. as already flushing 2024-11-20T08:59:30,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb. 2024-11-20T08:59:30,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-20T08:59:30,757 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 1c52b94a9e6adce80d2d034c31b67cfb, NAME => 'TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T08:59:30,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 1c52b94a9e6adce80d2d034c31b67cfb, NAME => 'TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T08:59:30,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1c52b94a9e6adce80d2d034c31b67cfb, NAME => 'TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1c52b94a9e6adce80d2d034c31b67cfb, NAME => 'TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T08:59:30,762 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/C/cf14c4a783214851a7a97272dc5bc134, entries=150, sequenceid=197, filesize=11.9 K 2024-11-20T08:59:30,764 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 1c52b94a9e6adce80d2d034c31b67cfb in 237ms, sequenceid=197, compaction requested=false 2024-11-20T08:59:30,765 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1c52b94a9e6adce80d2d034c31b67cfb: 2024-11-20T08:59:30,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(8855): Flush requested on 1c52b94a9e6adce80d2d034c31b67cfb 2024-11-20T08:59:30,881 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T08:59:30,882 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1c52b94a9e6adce80d2d034c31b67cfb 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-20T08:59:30,893 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/A/29b52998e7714e1dac22d2590d9e48db is 50, key is test_row_0/A:col10/1732093170549/Put/seqid=0 2024-11-20T08:59:30,911 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-20T08:59:30,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb. 2024-11-20T08:59:30,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb. as already flushing 2024-11-20T08:59:30,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb. 2024-11-20T08:59:30,912 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 1c52b94a9e6adce80d2d034c31b67cfb, NAME => 'TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T08:59:30,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 1c52b94a9e6adce80d2d034c31b67cfb, NAME => 'TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T08:59:30,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1c52b94a9e6adce80d2d034c31b67cfb, NAME => 'TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1c52b94a9e6adce80d2d034c31b67cfb, NAME => 'TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T08:59:30,916 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073741876_1052 (size=12151) 2024-11-20T08:59:30,920 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=212 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/A/29b52998e7714e1dac22d2590d9e48db 2024-11-20T08:59:30,942 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/B/c2af057f4aba4fceb12f7a79ba780730 is 50, key is test_row_0/B:col10/1732093170549/Put/seqid=0 2024-11-20T08:59:30,952 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:30,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43620 deadline: 1732093230945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:30,954 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:30,955 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:30,955 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:30,955 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:30,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43672 deadline: 1732093230948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:30,956 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:30,956 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:30,956 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:30,956 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:30,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43628 deadline: 1732093230948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:30,956 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:30,957 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:30,957 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:30,957 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:30,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43644 deadline: 1732093230950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:30,957 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:30,957 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:30,957 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:30,966 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073741877_1053 (size=12151) 2024-11-20T08:59:30,968 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=212 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/B/c2af057f4aba4fceb12f7a79ba780730 2024-11-20T08:59:30,982 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/C/47928dd6881a428d8a2086225812b0ee is 50, key is test_row_0/C:col10/1732093170549/Put/seqid=0 2024-11-20T08:59:30,996 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073741878_1054 (size=12151) 2024-11-20T08:59:30,997 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=212 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/C/47928dd6881a428d8a2086225812b0ee 2024-11-20T08:59:31,005 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/A/29b52998e7714e1dac22d2590d9e48db as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/29b52998e7714e1dac22d2590d9e48db 2024-11-20T08:59:31,016 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/29b52998e7714e1dac22d2590d9e48db, entries=150, sequenceid=212, filesize=11.9 K 2024-11-20T08:59:31,018 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/B/c2af057f4aba4fceb12f7a79ba780730 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/c2af057f4aba4fceb12f7a79ba780730 2024-11-20T08:59:31,033 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/c2af057f4aba4fceb12f7a79ba780730, entries=150, sequenceid=212, filesize=11.9 K 2024-11-20T08:59:31,035 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/C/47928dd6881a428d8a2086225812b0ee as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/C/47928dd6881a428d8a2086225812b0ee 2024-11-20T08:59:31,039 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/C/9f5278a2f6b3484ea4ed75021ab2baf8 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/C/9f5278a2f6b3484ea4ed75021ab2baf8 2024-11-20T08:59:31,052 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/C/47928dd6881a428d8a2086225812b0ee, entries=150, sequenceid=212, filesize=11.9 K 2024-11-20T08:59:31,054 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 1c52b94a9e6adce80d2d034c31b67cfb in 173ms, sequenceid=212, compaction requested=true 2024-11-20T08:59:31,054 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1c52b94a9e6adce80d2d034c31b67cfb: 2024-11-20T08:59:31,054 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1c52b94a9e6adce80d2d034c31b67cfb:A, priority=-2147483648, current under compaction store size is 2 2024-11-20T08:59:31,055 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T08:59:31,055 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1c52b94a9e6adce80d2d034c31b67cfb:B, priority=-2147483648, current under compaction store size is 3 2024-11-20T08:59:31,055 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-20T08:59:31,055 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1c52b94a9e6adce80d2d034c31b67cfb:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T08:59:31,055 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-20T08:59:31,055 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 3 compacting, 2 eligible, 16 blocking 2024-11-20T08:59:31,057 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-20T08:59:31,057 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-20T08:59:31,057 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb. because compaction request was cancelled 2024-11-20T08:59:31,057 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1c52b94a9e6adce80d2d034c31b67cfb:C 2024-11-20T08:59:31,057 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T08:59:31,058 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1c52b94a9e6adce80d2d034c31b67cfb/C of 1c52b94a9e6adce80d2d034c31b67cfb into 9f5278a2f6b3484ea4ed75021ab2baf8(size=12.2 K), total size for store is 36.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T08:59:31,058 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1c52b94a9e6adce80d2d034c31b67cfb: 2024-11-20T08:59:31,058 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., storeName=1c52b94a9e6adce80d2d034c31b67cfb/C, priority=13, startTime=1732093170453; duration=0sec 2024-11-20T08:59:31,058 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T08:59:31,058 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1c52b94a9e6adce80d2d034c31b67cfb:C 2024-11-20T08:59:31,058 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T08:59:31,060 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T08:59:31,060 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HStore(1541): 1c52b94a9e6adce80d2d034c31b67cfb/A is initiating minor compaction (all files) 2024-11-20T08:59:31,060 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1c52b94a9e6adce80d2d034c31b67cfb/A in TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb. 2024-11-20T08:59:31,060 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/f8265531d4f14044b8a1ad0cac34e7fe, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/b37f571b76584fbb8c4b6a159dad0bc8, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/29b52998e7714e1dac22d2590d9e48db] into tmpdir=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp, totalSize=36.0 K 2024-11-20T08:59:31,061 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T08:59:31,061 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1541): 1c52b94a9e6adce80d2d034c31b67cfb/B is initiating minor compaction (all files) 2024-11-20T08:59:31,061 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1c52b94a9e6adce80d2d034c31b67cfb/B in TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb. 2024-11-20T08:59:31,061 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/9799a5642bfe40259b188a37202879cb, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/d47461f002a442bba66ba2b1f5e06318, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/c2af057f4aba4fceb12f7a79ba780730] into tmpdir=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp, totalSize=36.0 K 2024-11-20T08:59:31,062 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.Compactor(225): Compacting f8265531d4f14044b8a1ad0cac34e7fe, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1732093169497 2024-11-20T08:59:31,062 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting 9799a5642bfe40259b188a37202879cb, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1732093169497 2024-11-20T08:59:31,063 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting d47461f002a442bba66ba2b1f5e06318, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1732093169882 2024-11-20T08:59:31,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(8855): Flush requested on 1c52b94a9e6adce80d2d034c31b67cfb 2024-11-20T08:59:31,063 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.Compactor(225): Compacting b37f571b76584fbb8c4b6a159dad0bc8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1732093169882 2024-11-20T08:59:31,063 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T08:59:31,064 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1c52b94a9e6adce80d2d034c31b67cfb 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-20T08:59:31,064 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting c2af057f4aba4fceb12f7a79ba780730, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1732093170534 2024-11-20T08:59:31,065 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.Compactor(225): Compacting 29b52998e7714e1dac22d2590d9e48db, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1732093170534 2024-11-20T08:59:31,069 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-20T08:59:31,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb. 2024-11-20T08:59:31,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb. as already flushing 2024-11-20T08:59:31,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb. 2024-11-20T08:59:31,069 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 1c52b94a9e6adce80d2d034c31b67cfb, NAME => 'TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T08:59:31,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 1c52b94a9e6adce80d2d034c31b67cfb, NAME => 'TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T08:59:31,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1c52b94a9e6adce80d2d034c31b67cfb, NAME => 'TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1c52b94a9e6adce80d2d034c31b67cfb, NAME => 'TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T08:59:31,079 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/A/16ddf979ca0144ca8d520c50c0efd664 is 50, key is test_row_0/A:col10/1732093170944/Put/seqid=0 2024-11-20T08:59:31,083 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1c52b94a9e6adce80d2d034c31b67cfb#B#compaction#43 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T08:59:31,084 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/B/0324d355654845f48a920fdd882fbc25 is 50, key is test_row_0/B:col10/1732093170549/Put/seqid=0 2024-11-20T08:59:31,089 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1c52b94a9e6adce80d2d034c31b67cfb#A#compaction#44 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T08:59:31,090 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/A/658de7bcce954012a0499e1becc615d1 is 50, key is test_row_0/A:col10/1732093170549/Put/seqid=0 2024-11-20T08:59:31,095 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073741880_1056 (size=12629) 2024-11-20T08:59:31,098 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073741879_1055 (size=14541) 2024-11-20T08:59:31,103 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/B/0324d355654845f48a920fdd882fbc25 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/0324d355654845f48a920fdd882fbc25 2024-11-20T08:59:31,115 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073741881_1057 (size=12629) 2024-11-20T08:59:31,123 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1c52b94a9e6adce80d2d034c31b67cfb/B of 1c52b94a9e6adce80d2d034c31b67cfb into 0324d355654845f48a920fdd882fbc25(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T08:59:31,123 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1c52b94a9e6adce80d2d034c31b67cfb: 2024-11-20T08:59:31,123 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., storeName=1c52b94a9e6adce80d2d034c31b67cfb/B, priority=13, startTime=1732093171055; duration=0sec 2024-11-20T08:59:31,123 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T08:59:31,123 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1c52b94a9e6adce80d2d034c31b67cfb:B 2024-11-20T08:59:31,143 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:31,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43628 deadline: 1732093231140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:31,144 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:31,144 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:31,144 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:31,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43620 deadline: 1732093231141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:31,145 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:31,145 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:31,145 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:31,145 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:31,145 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:31,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43644 deadline: 1732093231142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:31,146 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:31,146 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:31,146 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:31,146 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:31,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43672 deadline: 1732093231144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:31,146 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:31,146 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:31,146 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:31,225 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-20T08:59:31,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb. 2024-11-20T08:59:31,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb. as already flushing 2024-11-20T08:59:31,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb. 2024-11-20T08:59:31,225 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 1c52b94a9e6adce80d2d034c31b67cfb, NAME => 'TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T08:59:31,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 1c52b94a9e6adce80d2d034c31b67cfb, NAME => 'TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T08:59:31,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1c52b94a9e6adce80d2d034c31b67cfb, NAME => 'TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1c52b94a9e6adce80d2d034c31b67cfb, NAME => 'TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T08:59:31,253 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:31,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43620 deadline: 1732093231248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:31,255 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:31,255 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:31,255 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:31,255 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:31,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43628 deadline: 1732093231249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:31,255 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:31,255 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:31,256 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:31,260 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:31,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43644 deadline: 1732093231257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:31,261 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:31,261 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:31,261 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:31,261 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:31,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43672 deadline: 1732093231257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:31,262 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:31,262 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:31,262 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:31,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-20T08:59:31,380 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-20T08:59:31,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb. 2024-11-20T08:59:31,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb. as already flushing 2024-11-20T08:59:31,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb. 2024-11-20T08:59:31,381 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 1c52b94a9e6adce80d2d034c31b67cfb, NAME => 'TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T08:59:31,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 1c52b94a9e6adce80d2d034c31b67cfb, NAME => 'TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T08:59:31,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1c52b94a9e6adce80d2d034c31b67cfb, NAME => 'TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1c52b94a9e6adce80d2d034c31b67cfb, NAME => 'TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T08:59:31,461 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:31,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43620 deadline: 1732093231459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:31,462 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:31,462 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:31,462 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:31,472 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:31,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43644 deadline: 1732093231467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:31,473 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:31,473 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:31,473 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:31,473 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:31,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43628 deadline: 1732093231467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:31,473 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:31,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43672 deadline: 1732093231467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:31,475 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:31,475 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:31,475 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:31,475 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:31,475 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:31,475 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:31,500 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=237 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/A/16ddf979ca0144ca8d520c50c0efd664 2024-11-20T08:59:31,518 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/B/b1b43658f5c84c85b3e2c729ef0dd4f2 is 50, key is test_row_0/B:col10/1732093170944/Put/seqid=0 2024-11-20T08:59:31,527 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/A/658de7bcce954012a0499e1becc615d1 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/658de7bcce954012a0499e1becc615d1 2024-11-20T08:59:31,536 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-20T08:59:31,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb. 2024-11-20T08:59:31,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb. as already flushing 2024-11-20T08:59:31,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb. 2024-11-20T08:59:31,537 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 1c52b94a9e6adce80d2d034c31b67cfb, NAME => 'TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T08:59:31,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 1c52b94a9e6adce80d2d034c31b67cfb, NAME => 'TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T08:59:31,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1c52b94a9e6adce80d2d034c31b67cfb, NAME => 'TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1c52b94a9e6adce80d2d034c31b67cfb, NAME => 'TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T08:59:31,541 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073741882_1058 (size=12151) 2024-11-20T08:59:31,545 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1c52b94a9e6adce80d2d034c31b67cfb/A of 1c52b94a9e6adce80d2d034c31b67cfb into 658de7bcce954012a0499e1becc615d1(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T08:59:31,545 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1c52b94a9e6adce80d2d034c31b67cfb: 2024-11-20T08:59:31,545 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., storeName=1c52b94a9e6adce80d2d034c31b67cfb/A, priority=13, startTime=1732093171054; duration=0sec 2024-11-20T08:59:31,546 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T08:59:31,546 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1c52b94a9e6adce80d2d034c31b67cfb:A 2024-11-20T08:59:31,692 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-20T08:59:31,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb. 2024-11-20T08:59:31,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb. as already flushing 2024-11-20T08:59:31,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb. 2024-11-20T08:59:31,692 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 1c52b94a9e6adce80d2d034c31b67cfb, NAME => 'TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T08:59:31,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 1c52b94a9e6adce80d2d034c31b67cfb, NAME => 'TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T08:59:31,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1c52b94a9e6adce80d2d034c31b67cfb, NAME => 'TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1c52b94a9e6adce80d2d034c31b67cfb, NAME => 'TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T08:59:31,771 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:31,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43620 deadline: 1732093231768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:31,772 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:31,772 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:31,772 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:31,777 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:31,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43644 deadline: 1732093231777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:31,778 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:31,778 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:31,778 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:31,790 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:31,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43672 deadline: 1732093231788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:31,791 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:31,791 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:31,791 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:31,791 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:31,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43628 deadline: 1732093231788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:31,791 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:31,791 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:31,791 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:31,849 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-20T08:59:31,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb. 2024-11-20T08:59:31,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb. as already flushing 2024-11-20T08:59:31,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb. 2024-11-20T08:59:31,850 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 1c52b94a9e6adce80d2d034c31b67cfb, NAME => 'TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T08:59:31,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 1c52b94a9e6adce80d2d034c31b67cfb, NAME => 'TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T08:59:31,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1c52b94a9e6adce80d2d034c31b67cfb, NAME => 'TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1c52b94a9e6adce80d2d034c31b67cfb, NAME => 'TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T08:59:31,943 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=237 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/B/b1b43658f5c84c85b3e2c729ef0dd4f2 2024-11-20T08:59:31,962 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/C/e7e8076243c74ec1ab9cb9bb438b8166 is 50, key is test_row_0/C:col10/1732093170944/Put/seqid=0 2024-11-20T08:59:31,978 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073741883_1059 (size=12151) 2024-11-20T08:59:31,979 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=237 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/C/e7e8076243c74ec1ab9cb9bb438b8166 2024-11-20T08:59:31,993 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/A/16ddf979ca0144ca8d520c50c0efd664 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/16ddf979ca0144ca8d520c50c0efd664 2024-11-20T08:59:32,003 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-20T08:59:32,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb. 2024-11-20T08:59:32,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb. as already flushing 2024-11-20T08:59:32,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb. 2024-11-20T08:59:32,004 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 1c52b94a9e6adce80d2d034c31b67cfb, NAME => 'TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T08:59:32,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 1c52b94a9e6adce80d2d034c31b67cfb, NAME => 'TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T08:59:32,005 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/16ddf979ca0144ca8d520c50c0efd664, entries=200, sequenceid=237, filesize=14.2 K 2024-11-20T08:59:32,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1c52b94a9e6adce80d2d034c31b67cfb, NAME => 'TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1c52b94a9e6adce80d2d034c31b67cfb, NAME => 'TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T08:59:32,007 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/B/b1b43658f5c84c85b3e2c729ef0dd4f2 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/b1b43658f5c84c85b3e2c729ef0dd4f2 2024-11-20T08:59:32,024 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/b1b43658f5c84c85b3e2c729ef0dd4f2, entries=150, sequenceid=237, filesize=11.9 K 2024-11-20T08:59:32,026 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/C/e7e8076243c74ec1ab9cb9bb438b8166 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/C/e7e8076243c74ec1ab9cb9bb438b8166 2024-11-20T08:59:32,049 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/C/e7e8076243c74ec1ab9cb9bb438b8166, entries=150, sequenceid=237, filesize=11.9 K 2024-11-20T08:59:32,050 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 1c52b94a9e6adce80d2d034c31b67cfb in 987ms, sequenceid=237, compaction requested=true 2024-11-20T08:59:32,050 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1c52b94a9e6adce80d2d034c31b67cfb: 2024-11-20T08:59:32,051 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1c52b94a9e6adce80d2d034c31b67cfb:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T08:59:32,051 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T08:59:32,051 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-20T08:59:32,051 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1c52b94a9e6adce80d2d034c31b67cfb:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T08:59:32,051 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T08:59:32,051 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-20T08:59:32,051 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1c52b94a9e6adce80d2d034c31b67cfb:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T08:59:32,051 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T08:59:32,052 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-20T08:59:32,052 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-20T08:59:32,052 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb. because compaction request was cancelled 2024-11-20T08:59:32,052 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1c52b94a9e6adce80d2d034c31b67cfb:B 2024-11-20T08:59:32,052 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T08:59:32,053 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-20T08:59:32,053 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-20T08:59:32,053 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb. because compaction request was cancelled 2024-11-20T08:59:32,053 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1c52b94a9e6adce80d2d034c31b67cfb:A 2024-11-20T08:59:32,054 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48980 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T08:59:32,055 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1541): 1c52b94a9e6adce80d2d034c31b67cfb/C is initiating minor compaction (all files) 2024-11-20T08:59:32,055 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1c52b94a9e6adce80d2d034c31b67cfb/C in TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb. 2024-11-20T08:59:32,055 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/C/9f5278a2f6b3484ea4ed75021ab2baf8, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/C/cf14c4a783214851a7a97272dc5bc134, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/C/47928dd6881a428d8a2086225812b0ee, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/C/e7e8076243c74ec1ab9cb9bb438b8166] into tmpdir=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp, totalSize=47.8 K 2024-11-20T08:59:32,058 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting 9f5278a2f6b3484ea4ed75021ab2baf8, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1732093169497 2024-11-20T08:59:32,058 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting cf14c4a783214851a7a97272dc5bc134, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1732093169882 2024-11-20T08:59:32,059 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting 47928dd6881a428d8a2086225812b0ee, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1732093170534 2024-11-20T08:59:32,059 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting e7e8076243c74ec1ab9cb9bb438b8166, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1732093170944 2024-11-20T08:59:32,079 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1c52b94a9e6adce80d2d034c31b67cfb#C#compaction#47 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T08:59:32,080 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/C/a7229eb625524e8cac1d13ef0ee97a9e is 50, key is test_row_0/C:col10/1732093170944/Put/seqid=0 2024-11-20T08:59:32,099 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073741884_1060 (size=12663) 2024-11-20T08:59:32,112 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/C/a7229eb625524e8cac1d13ef0ee97a9e as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/C/a7229eb625524e8cac1d13ef0ee97a9e 2024-11-20T08:59:32,124 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1c52b94a9e6adce80d2d034c31b67cfb/C of 1c52b94a9e6adce80d2d034c31b67cfb into a7229eb625524e8cac1d13ef0ee97a9e(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T08:59:32,124 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1c52b94a9e6adce80d2d034c31b67cfb: 2024-11-20T08:59:32,124 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., storeName=1c52b94a9e6adce80d2d034c31b67cfb/C, priority=12, startTime=1732093172051; duration=0sec 2024-11-20T08:59:32,124 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T08:59:32,124 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1c52b94a9e6adce80d2d034c31b67cfb:C 2024-11-20T08:59:32,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(8855): Flush requested on 1c52b94a9e6adce80d2d034c31b67cfb 2024-11-20T08:59:32,148 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T08:59:32,148 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1c52b94a9e6adce80d2d034c31b67cfb 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-20T08:59:32,159 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-20T08:59:32,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb. 2024-11-20T08:59:32,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb. as already flushing 2024-11-20T08:59:32,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb. 2024-11-20T08:59:32,160 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 1c52b94a9e6adce80d2d034c31b67cfb, NAME => 'TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T08:59:32,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 1c52b94a9e6adce80d2d034c31b67cfb, NAME => 'TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T08:59:32,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1c52b94a9e6adce80d2d034c31b67cfb, NAME => 'TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1c52b94a9e6adce80d2d034c31b67cfb, NAME => 'TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T08:59:32,164 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/A/7c6a5afbecb74501a3c14b41639b7d00 is 50, key is test_row_0/A:col10/1732093171137/Put/seqid=0 2024-11-20T08:59:32,183 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073741885_1061 (size=14541) 2024-11-20T08:59:32,188 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=253 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/A/7c6a5afbecb74501a3c14b41639b7d00 2024-11-20T08:59:32,205 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/B/c0351c0dd773448289ffb124f0c87981 is 50, key is test_row_0/B:col10/1732093171137/Put/seqid=0 2024-11-20T08:59:32,249 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073741886_1062 (size=12151) 2024-11-20T08:59:32,250 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=253 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/B/c0351c0dd773448289ffb124f0c87981 2024-11-20T08:59:32,264 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/C/a083bcdfec7342809d567f1cb5f5123b is 50, key is test_row_0/C:col10/1732093171137/Put/seqid=0 2024-11-20T08:59:32,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-20T08:59:32,286 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073741887_1063 (size=12151) 2024-11-20T08:59:32,291 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:32,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43620 deadline: 1732093232287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:32,293 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:32,293 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:32,293 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:32,297 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:32,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43608 deadline: 1732093232292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:32,298 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:32,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43644 deadline: 1732093232295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:32,299 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:32,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43628 deadline: 1732093232297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:32,299 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:32,299 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:32,300 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:32,300 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:32,301 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:32,301 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:32,301 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:32,304 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:32,304 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:32,304 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:32,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43672 deadline: 1732093232300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:32,305 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:32,305 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:32,305 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:32,316 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-20T08:59:32,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb. 2024-11-20T08:59:32,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb. as already flushing 2024-11-20T08:59:32,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb. 2024-11-20T08:59:32,316 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 1c52b94a9e6adce80d2d034c31b67cfb, NAME => 'TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T08:59:32,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 1c52b94a9e6adce80d2d034c31b67cfb, NAME => 'TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T08:59:32,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1c52b94a9e6adce80d2d034c31b67cfb, NAME => 'TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1c52b94a9e6adce80d2d034c31b67cfb, NAME => 'TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T08:59:32,409 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:32,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43644 deadline: 1732093232407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:32,409 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:32,410 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:32,410 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:32,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43608 deadline: 1732093232407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:32,410 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:32,410 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:32,410 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:32,410 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:32,471 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-20T08:59:32,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb. 2024-11-20T08:59:32,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb. as already flushing 2024-11-20T08:59:32,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb. 2024-11-20T08:59:32,472 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 1c52b94a9e6adce80d2d034c31b67cfb, NAME => 'TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T08:59:32,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 1c52b94a9e6adce80d2d034c31b67cfb, NAME => 'TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T08:59:32,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1c52b94a9e6adce80d2d034c31b67cfb, NAME => 'TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1c52b94a9e6adce80d2d034c31b67cfb, NAME => 'TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T08:59:32,620 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:32,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43644 deadline: 1732093232617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:32,621 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:32,621 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:32,621 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:32,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43608 deadline: 1732093232617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:32,621 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:32,621 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:32,622 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:32,622 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:32,627 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-20T08:59:32,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb. 2024-11-20T08:59:32,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb. as already flushing 2024-11-20T08:59:32,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb. 2024-11-20T08:59:32,627 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 1c52b94a9e6adce80d2d034c31b67cfb, NAME => 'TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T08:59:32,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 1c52b94a9e6adce80d2d034c31b67cfb, NAME => 'TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T08:59:32,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1c52b94a9e6adce80d2d034c31b67cfb, NAME => 'TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1c52b94a9e6adce80d2d034c31b67cfb, NAME => 'TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T08:59:32,687 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=253 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/C/a083bcdfec7342809d567f1cb5f5123b 2024-11-20T08:59:32,695 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/A/7c6a5afbecb74501a3c14b41639b7d00 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/7c6a5afbecb74501a3c14b41639b7d00 2024-11-20T08:59:32,703 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/7c6a5afbecb74501a3c14b41639b7d00, entries=200, sequenceid=253, filesize=14.2 K 2024-11-20T08:59:32,704 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/B/c0351c0dd773448289ffb124f0c87981 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/c0351c0dd773448289ffb124f0c87981 2024-11-20T08:59:32,713 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/c0351c0dd773448289ffb124f0c87981, entries=150, sequenceid=253, filesize=11.9 K 2024-11-20T08:59:32,714 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/C/a083bcdfec7342809d567f1cb5f5123b as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/C/a083bcdfec7342809d567f1cb5f5123b 2024-11-20T08:59:32,723 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/C/a083bcdfec7342809d567f1cb5f5123b, entries=150, sequenceid=253, filesize=11.9 K 2024-11-20T08:59:32,724 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=140.89 KB/144270 for 1c52b94a9e6adce80d2d034c31b67cfb in 576ms, sequenceid=253, compaction requested=true 2024-11-20T08:59:32,724 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1c52b94a9e6adce80d2d034c31b67cfb: 2024-11-20T08:59:32,725 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1c52b94a9e6adce80d2d034c31b67cfb:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T08:59:32,725 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T08:59:32,725 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T08:59:32,725 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1c52b94a9e6adce80d2d034c31b67cfb:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T08:59:32,725 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T08:59:32,725 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T08:59:32,725 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1c52b94a9e6adce80d2d034c31b67cfb:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T08:59:32,725 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T08:59:32,728 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 41711 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T08:59:32,728 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HStore(1541): 1c52b94a9e6adce80d2d034c31b67cfb/A is initiating minor compaction (all files) 2024-11-20T08:59:32,728 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1c52b94a9e6adce80d2d034c31b67cfb/A in TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb. 2024-11-20T08:59:32,728 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/658de7bcce954012a0499e1becc615d1, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/16ddf979ca0144ca8d520c50c0efd664, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/7c6a5afbecb74501a3c14b41639b7d00] into tmpdir=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp, totalSize=40.7 K 2024-11-20T08:59:32,728 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36931 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T08:59:32,729 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1541): 1c52b94a9e6adce80d2d034c31b67cfb/B is initiating minor compaction (all files) 2024-11-20T08:59:32,729 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1c52b94a9e6adce80d2d034c31b67cfb/B in TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb. 2024-11-20T08:59:32,729 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/0324d355654845f48a920fdd882fbc25, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/b1b43658f5c84c85b3e2c729ef0dd4f2, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/c0351c0dd773448289ffb124f0c87981] into tmpdir=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp, totalSize=36.1 K 2024-11-20T08:59:32,729 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting 0324d355654845f48a920fdd882fbc25, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1732093170534 2024-11-20T08:59:32,729 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.Compactor(225): Compacting 658de7bcce954012a0499e1becc615d1, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1732093170534 2024-11-20T08:59:32,730 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.Compactor(225): Compacting 16ddf979ca0144ca8d520c50c0efd664, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1732093170944 2024-11-20T08:59:32,730 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting b1b43658f5c84c85b3e2c729ef0dd4f2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1732093170944 2024-11-20T08:59:32,730 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting c0351c0dd773448289ffb124f0c87981, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1732093171137 2024-11-20T08:59:32,731 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7c6a5afbecb74501a3c14b41639b7d00, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1732093171137 2024-11-20T08:59:32,746 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1c52b94a9e6adce80d2d034c31b67cfb#B#compaction#52 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-20T08:59:32,746 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1c52b94a9e6adce80d2d034c31b67cfb#A#compaction#51 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T08:59:32,747 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/B/c1a34a6a5f28497589cce7a85a483073 is 50, key is test_row_0/B:col10/1732093171137/Put/seqid=0 2024-11-20T08:59:32,747 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/A/feb70ce5513c4de4b8b3ebdf258e8e10 is 50, key is test_row_0/A:col10/1732093171137/Put/seqid=0 2024-11-20T08:59:32,758 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073741888_1064 (size=12731) 2024-11-20T08:59:32,768 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/A/feb70ce5513c4de4b8b3ebdf258e8e10 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/feb70ce5513c4de4b8b3ebdf258e8e10 2024-11-20T08:59:32,769 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073741889_1065 (size=12731) 2024-11-20T08:59:32,781 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-20T08:59:32,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb. 2024-11-20T08:59:32,782 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing 1c52b94a9e6adce80d2d034c31b67cfb 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-20T08:59:32,789 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/B/c1a34a6a5f28497589cce7a85a483073 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/c1a34a6a5f28497589cce7a85a483073 2024-11-20T08:59:32,794 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1c52b94a9e6adce80d2d034c31b67cfb/A of 1c52b94a9e6adce80d2d034c31b67cfb into feb70ce5513c4de4b8b3ebdf258e8e10(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T08:59:32,794 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1c52b94a9e6adce80d2d034c31b67cfb: 2024-11-20T08:59:32,795 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., storeName=1c52b94a9e6adce80d2d034c31b67cfb/A, priority=13, startTime=1732093172725; duration=0sec 2024-11-20T08:59:32,795 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T08:59:32,795 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1c52b94a9e6adce80d2d034c31b67cfb:A 2024-11-20T08:59:32,796 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-20T08:59:32,800 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1c52b94a9e6adce80d2d034c31b67cfb/B of 1c52b94a9e6adce80d2d034c31b67cfb into c1a34a6a5f28497589cce7a85a483073(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T08:59:32,800 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1c52b94a9e6adce80d2d034c31b67cfb: 2024-11-20T08:59:32,800 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., storeName=1c52b94a9e6adce80d2d034c31b67cfb/B, priority=13, startTime=1732093172725; duration=0sec 2024-11-20T08:59:32,800 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T08:59:32,800 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1c52b94a9e6adce80d2d034c31b67cfb:B 2024-11-20T08:59:32,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/A/8ea1b3fbfc494228bef8f4cf3b493f9b is 50, key is test_row_0/A:col10/1732093172291/Put/seqid=0 2024-11-20T08:59:32,801 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-20T08:59:32,801 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-20T08:59:32,801 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb. because compaction request was cancelled 2024-11-20T08:59:32,802 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1c52b94a9e6adce80d2d034c31b67cfb:C 2024-11-20T08:59:32,809 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073741890_1066 (size=12301) 2024-11-20T08:59:32,809 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=277 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/A/8ea1b3fbfc494228bef8f4cf3b493f9b 2024-11-20T08:59:32,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/B/bf5ed9748a4c4803ab40bc20f78082be is 50, key is test_row_0/B:col10/1732093172291/Put/seqid=0 2024-11-20T08:59:32,837 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073741891_1067 (size=12301) 2024-11-20T08:59:32,839 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=277 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/B/bf5ed9748a4c4803ab40bc20f78082be 2024-11-20T08:59:32,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/C/42168416d26a4307abf7c9dcdbc616bb is 50, key is test_row_0/C:col10/1732093172291/Put/seqid=0 2024-11-20T08:59:32,865 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073741892_1068 (size=12301) 2024-11-20T08:59:32,866 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=277 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/C/42168416d26a4307abf7c9dcdbc616bb 2024-11-20T08:59:32,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/A/8ea1b3fbfc494228bef8f4cf3b493f9b as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/8ea1b3fbfc494228bef8f4cf3b493f9b 2024-11-20T08:59:32,883 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/8ea1b3fbfc494228bef8f4cf3b493f9b, entries=150, sequenceid=277, filesize=12.0 K 2024-11-20T08:59:32,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/B/bf5ed9748a4c4803ab40bc20f78082be as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/bf5ed9748a4c4803ab40bc20f78082be 2024-11-20T08:59:32,894 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/bf5ed9748a4c4803ab40bc20f78082be, entries=150, sequenceid=277, filesize=12.0 K 2024-11-20T08:59:32,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/C/42168416d26a4307abf7c9dcdbc616bb as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/C/42168416d26a4307abf7c9dcdbc616bb 2024-11-20T08:59:32,904 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/C/42168416d26a4307abf7c9dcdbc616bb, entries=150, sequenceid=277, filesize=12.0 K 2024-11-20T08:59:32,912 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=0 B/0 for 1c52b94a9e6adce80d2d034c31b67cfb in 130ms, sequenceid=277, compaction requested=true 2024-11-20T08:59:32,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for 1c52b94a9e6adce80d2d034c31b67cfb: 2024-11-20T08:59:32,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb. 2024-11-20T08:59:32,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-11-20T08:59:32,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-11-20T08:59:32,921 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-11-20T08:59:32,921 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7790 sec 2024-11-20T08:59:32,928 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees in 2.7950 sec 2024-11-20T08:59:32,952 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T08:59:32,953 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1c52b94a9e6adce80d2d034c31b67cfb 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-20T08:59:32,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(8855): Flush requested on 1c52b94a9e6adce80d2d034c31b67cfb 2024-11-20T08:59:32,964 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/A/c496b5351b6544e4afadf75baa9478b7 is 50, key is test_row_0/A:col10/1732093172947/Put/seqid=0 2024-11-20T08:59:32,990 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073741893_1069 (size=17181) 2024-11-20T08:59:32,991 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=291 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/A/c496b5351b6544e4afadf75baa9478b7 2024-11-20T08:59:33,008 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/B/5f75c8c4487d4d47888476a57f38256b is 50, key is test_row_0/B:col10/1732093172947/Put/seqid=0 2024-11-20T08:59:33,021 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073741894_1070 (size=12301) 2024-11-20T08:59:33,071 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:33,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43644 deadline: 1732093233066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:33,072 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:33,072 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:33,073 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:33,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43608 deadline: 1732093233066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:33,073 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:33,073 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:33,073 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:33,073 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:33,182 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:33,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43608 deadline: 1732093233177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:33,183 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:33,183 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:33,184 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:33,181 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:33,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43644 deadline: 1732093233177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:33,184 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:33,184 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:33,184 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:33,313 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:33,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43620 deadline: 1732093233307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:33,316 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:33,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43628 deadline: 1732093233307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:33,320 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to be1eb2620e0e,34223,1732093159960 for 'test_row_2' in 1c52b94a9e6adce80d2d034c31b67cfb of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2179 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-20T08:59:33,321 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:33,321 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:33,320 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to be1eb2620e0e,34223,1732093159960 for 'test_row_0' in 1c52b94a9e6adce80d2d034c31b67cfb of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2181 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-20T08:59:33,321 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:33,321 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:33,321 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:33,321 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:33,326 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:33,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43672 deadline: 1732093233318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:33,327 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to be1eb2620e0e,34223,1732093159960 for 'test_row_2' in 1c52b94a9e6adce80d2d034c31b67cfb of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2183 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-20T08:59:33,328 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:33,328 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:33,328 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:33,394 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:33,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43608 deadline: 1732093233387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:33,394 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:33,395 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:33,395 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:33,395 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:33,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43644 deadline: 1732093233387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:33,395 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:33,395 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:33,395 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:33,424 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=291 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/B/5f75c8c4487d4d47888476a57f38256b 2024-11-20T08:59:33,439 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/C/97f6e608690c480fa86d22ec6cf11550 is 50, key is test_row_0/C:col10/1732093172947/Put/seqid=0 2024-11-20T08:59:33,469 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073741895_1071 (size=12301) 2024-11-20T08:59:33,708 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:33,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43608 deadline: 1732093233708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:33,709 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:33,709 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:33,709 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:33,709 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:33,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43644 deadline: 1732093233708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:33,710 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:33,710 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:33,710 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:33,871 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=291 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/C/97f6e608690c480fa86d22ec6cf11550 2024-11-20T08:59:33,884 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/A/c496b5351b6544e4afadf75baa9478b7 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/c496b5351b6544e4afadf75baa9478b7 2024-11-20T08:59:33,897 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/c496b5351b6544e4afadf75baa9478b7, entries=250, sequenceid=291, filesize=16.8 K 2024-11-20T08:59:33,899 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/B/5f75c8c4487d4d47888476a57f38256b as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/5f75c8c4487d4d47888476a57f38256b 2024-11-20T08:59:33,916 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/5f75c8c4487d4d47888476a57f38256b, entries=150, sequenceid=291, filesize=12.0 K 2024-11-20T08:59:33,918 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/C/97f6e608690c480fa86d22ec6cf11550 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/C/97f6e608690c480fa86d22ec6cf11550 2024-11-20T08:59:33,927 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/C/97f6e608690c480fa86d22ec6cf11550, entries=150, sequenceid=291, filesize=12.0 K 2024-11-20T08:59:33,929 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 1c52b94a9e6adce80d2d034c31b67cfb in 977ms, sequenceid=291, compaction requested=true 2024-11-20T08:59:33,929 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1c52b94a9e6adce80d2d034c31b67cfb: 2024-11-20T08:59:33,929 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1c52b94a9e6adce80d2d034c31b67cfb:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T08:59:33,930 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T08:59:33,930 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T08:59:33,930 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T08:59:33,930 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1c52b94a9e6adce80d2d034c31b67cfb:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T08:59:33,930 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T08:59:33,930 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1c52b94a9e6adce80d2d034c31b67cfb:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T08:59:33,930 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T08:59:33,931 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42213 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T08:59:33,932 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HStore(1541): 1c52b94a9e6adce80d2d034c31b67cfb/A is initiating minor compaction (all files) 2024-11-20T08:59:33,932 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1c52b94a9e6adce80d2d034c31b67cfb/A in TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb. 2024-11-20T08:59:33,932 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/feb70ce5513c4de4b8b3ebdf258e8e10, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/8ea1b3fbfc494228bef8f4cf3b493f9b, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/c496b5351b6544e4afadf75baa9478b7] into tmpdir=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp, totalSize=41.2 K 2024-11-20T08:59:33,933 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.Compactor(225): Compacting feb70ce5513c4de4b8b3ebdf258e8e10, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1732093171137 2024-11-20T08:59:33,935 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37333 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T08:59:33,935 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1541): 1c52b94a9e6adce80d2d034c31b67cfb/B is initiating minor compaction (all files) 2024-11-20T08:59:33,935 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1c52b94a9e6adce80d2d034c31b67cfb/B in TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb. 2024-11-20T08:59:33,935 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/c1a34a6a5f28497589cce7a85a483073, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/bf5ed9748a4c4803ab40bc20f78082be, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/5f75c8c4487d4d47888476a57f38256b] into tmpdir=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp, totalSize=36.5 K 2024-11-20T08:59:33,935 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8ea1b3fbfc494228bef8f4cf3b493f9b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=277, earliestPutTs=1732093172254 2024-11-20T08:59:33,936 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting c1a34a6a5f28497589cce7a85a483073, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1732093171137 2024-11-20T08:59:33,937 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.Compactor(225): Compacting c496b5351b6544e4afadf75baa9478b7, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1732093172935 2024-11-20T08:59:33,937 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting bf5ed9748a4c4803ab40bc20f78082be, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=277, earliestPutTs=1732093172254 2024-11-20T08:59:33,938 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting 5f75c8c4487d4d47888476a57f38256b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1732093172944 2024-11-20T08:59:33,957 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1c52b94a9e6adce80d2d034c31b67cfb#B#compaction#60 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-20T08:59:33,958 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/B/80982404f8a746e496f91705020a9fed is 50, key is test_row_0/B:col10/1732093172947/Put/seqid=0 2024-11-20T08:59:33,960 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1c52b94a9e6adce80d2d034c31b67cfb#A#compaction#59 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T08:59:33,961 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/A/9fcdd67484d546e2ae02bbbacc79851b is 50, key is test_row_0/A:col10/1732093172947/Put/seqid=0 2024-11-20T08:59:33,974 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073741896_1072 (size=12983) 2024-11-20T08:59:33,983 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/B/80982404f8a746e496f91705020a9fed as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/80982404f8a746e496f91705020a9fed 2024-11-20T08:59:33,997 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073741897_1073 (size=12983) 2024-11-20T08:59:33,997 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1c52b94a9e6adce80d2d034c31b67cfb/B of 1c52b94a9e6adce80d2d034c31b67cfb into 80982404f8a746e496f91705020a9fed(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T08:59:33,998 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1c52b94a9e6adce80d2d034c31b67cfb: 2024-11-20T08:59:33,998 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., storeName=1c52b94a9e6adce80d2d034c31b67cfb/B, priority=13, startTime=1732093173930; duration=0sec 2024-11-20T08:59:33,998 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T08:59:33,998 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1c52b94a9e6adce80d2d034c31b67cfb:B 2024-11-20T08:59:33,998 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T08:59:34,002 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49416 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T08:59:34,004 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1541): 1c52b94a9e6adce80d2d034c31b67cfb/C is initiating minor compaction (all files) 2024-11-20T08:59:34,004 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1c52b94a9e6adce80d2d034c31b67cfb/C in TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb. 2024-11-20T08:59:34,005 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/C/a7229eb625524e8cac1d13ef0ee97a9e, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/C/a083bcdfec7342809d567f1cb5f5123b, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/C/42168416d26a4307abf7c9dcdbc616bb, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/C/97f6e608690c480fa86d22ec6cf11550] into tmpdir=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp, totalSize=48.3 K 2024-11-20T08:59:34,006 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/A/9fcdd67484d546e2ae02bbbacc79851b as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/9fcdd67484d546e2ae02bbbacc79851b 2024-11-20T08:59:34,016 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1c52b94a9e6adce80d2d034c31b67cfb/A of 1c52b94a9e6adce80d2d034c31b67cfb into 9fcdd67484d546e2ae02bbbacc79851b(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T08:59:34,016 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1c52b94a9e6adce80d2d034c31b67cfb: 2024-11-20T08:59:34,016 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., storeName=1c52b94a9e6adce80d2d034c31b67cfb/A, priority=13, startTime=1732093173929; duration=0sec 2024-11-20T08:59:34,017 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T08:59:34,017 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1c52b94a9e6adce80d2d034c31b67cfb:A 2024-11-20T08:59:34,024 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting a7229eb625524e8cac1d13ef0ee97a9e, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1732093170944 2024-11-20T08:59:34,025 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting a083bcdfec7342809d567f1cb5f5123b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1732093171137 2024-11-20T08:59:34,026 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting 42168416d26a4307abf7c9dcdbc616bb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=277, earliestPutTs=1732093172254 2024-11-20T08:59:34,028 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting 97f6e608690c480fa86d22ec6cf11550, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1732093172944 2024-11-20T08:59:34,041 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1c52b94a9e6adce80d2d034c31b67cfb#C#compaction#61 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T08:59:34,042 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/C/c586a4522ac94f06a72e6ce750175160 is 50, key is test_row_0/C:col10/1732093172947/Put/seqid=0 2024-11-20T08:59:34,047 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073741898_1074 (size=12949) 2024-11-20T08:59:34,057 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/C/c586a4522ac94f06a72e6ce750175160 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/C/c586a4522ac94f06a72e6ce750175160 2024-11-20T08:59:34,065 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1c52b94a9e6adce80d2d034c31b67cfb/C of 1c52b94a9e6adce80d2d034c31b67cfb into c586a4522ac94f06a72e6ce750175160(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T08:59:34,065 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1c52b94a9e6adce80d2d034c31b67cfb: 2024-11-20T08:59:34,065 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., storeName=1c52b94a9e6adce80d2d034c31b67cfb/C, priority=12, startTime=1732093173930; duration=0sec 2024-11-20T08:59:34,066 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T08:59:34,066 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1c52b94a9e6adce80d2d034c31b67cfb:C 2024-11-20T08:59:34,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(8855): Flush requested on 1c52b94a9e6adce80d2d034c31b67cfb 2024-11-20T08:59:34,223 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T08:59:34,223 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1c52b94a9e6adce80d2d034c31b67cfb 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-20T08:59:34,230 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/A/7ab3be557dbf4ae0a380a5d28adf2d18 is 50, key is test_row_0/A:col10/1732093174219/Put/seqid=0 2024-11-20T08:59:34,238 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073741899_1075 (size=12301) 2024-11-20T08:59:34,253 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:34,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43644 deadline: 1732093234250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:34,253 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:34,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43608 deadline: 1732093234250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:34,254 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:34,254 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:34,254 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:34,254 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:34,254 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:34,254 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:34,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-20T08:59:34,287 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T08:59:34,288 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T08:59:34,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees 2024-11-20T08:59:34,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-20T08:59:34,293 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T08:59:34,294 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T08:59:34,294 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T08:59:34,359 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:34,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43644 deadline: 1732093234357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:34,359 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:34,359 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:34,359 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:34,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43608 deadline: 1732093234357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:34,360 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:34,360 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:34,360 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:34,360 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:34,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-20T08:59:34,447 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-20T08:59:34,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb. 2024-11-20T08:59:34,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb. as already flushing 2024-11-20T08:59:34,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb. 2024-11-20T08:59:34,447 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 1c52b94a9e6adce80d2d034c31b67cfb, NAME => 'TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T08:59:34,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 1c52b94a9e6adce80d2d034c31b67cfb, NAME => 'TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T08:59:34,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1c52b94a9e6adce80d2d034c31b67cfb, NAME => 'TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1c52b94a9e6adce80d2d034c31b67cfb, NAME => 'TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T08:59:34,567 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:34,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43644 deadline: 1732093234567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:34,568 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:34,568 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:34,568 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:34,568 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:34,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43608 deadline: 1732093234567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:34,569 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:34,569 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:34,569 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:34,601 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-20T08:59:34,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb. 2024-11-20T08:59:34,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb. as already flushing 2024-11-20T08:59:34,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb. 2024-11-20T08:59:34,604 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 1c52b94a9e6adce80d2d034c31b67cfb, NAME => 'TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T08:59:34,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 1c52b94a9e6adce80d2d034c31b67cfb, NAME => 'TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T08:59:34,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1c52b94a9e6adce80d2d034c31b67cfb, NAME => 'TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1c52b94a9e6adce80d2d034c31b67cfb, NAME => 'TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T08:59:34,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-20T08:59:34,640 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=320 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/A/7ab3be557dbf4ae0a380a5d28adf2d18 2024-11-20T08:59:34,658 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/B/8c563d2983a34d87a1dcc4db311791a6 is 50, key is test_row_0/B:col10/1732093174219/Put/seqid=0 2024-11-20T08:59:34,669 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073741900_1076 (size=12301) 2024-11-20T08:59:34,757 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-20T08:59:34,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb. 2024-11-20T08:59:34,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb. as already flushing 2024-11-20T08:59:34,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb. 2024-11-20T08:59:34,758 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 1c52b94a9e6adce80d2d034c31b67cfb, NAME => 'TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T08:59:34,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 1c52b94a9e6adce80d2d034c31b67cfb, NAME => 'TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T08:59:34,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1c52b94a9e6adce80d2d034c31b67cfb, NAME => 'TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1c52b94a9e6adce80d2d034c31b67cfb, NAME => 'TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T08:59:34,880 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:34,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43608 deadline: 1732093234877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:34,881 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:34,881 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:34,881 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:34,881 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:34,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43644 deadline: 1732093234877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:34,881 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:34,881 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:34,881 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:34,912 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-20T08:59:34,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb. 2024-11-20T08:59:34,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb. as already flushing 2024-11-20T08:59:34,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb. 2024-11-20T08:59:34,913 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 1c52b94a9e6adce80d2d034c31b67cfb, NAME => 'TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T08:59:34,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 1c52b94a9e6adce80d2d034c31b67cfb, NAME => 'TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T08:59:34,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1c52b94a9e6adce80d2d034c31b67cfb, NAME => 'TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1c52b94a9e6adce80d2d034c31b67cfb, NAME => 'TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T08:59:34,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-20T08:59:35,067 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-20T08:59:35,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb. 2024-11-20T08:59:35,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb. as already flushing 2024-11-20T08:59:35,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb. 2024-11-20T08:59:35,068 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 1c52b94a9e6adce80d2d034c31b67cfb, NAME => 'TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T08:59:35,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 1c52b94a9e6adce80d2d034c31b67cfb, NAME => 'TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T08:59:35,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1c52b94a9e6adce80d2d034c31b67cfb, NAME => 'TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1c52b94a9e6adce80d2d034c31b67cfb, NAME => 'TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T08:59:35,071 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=320 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/B/8c563d2983a34d87a1dcc4db311791a6 2024-11-20T08:59:35,086 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/C/48e572da1fb345e1b8ac33095a43d011 is 50, key is test_row_0/C:col10/1732093174219/Put/seqid=0 2024-11-20T08:59:35,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073741901_1077 (size=12301) 2024-11-20T08:59:35,223 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-20T08:59:35,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb. 2024-11-20T08:59:35,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb. as already flushing 2024-11-20T08:59:35,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb. 2024-11-20T08:59:35,224 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 1c52b94a9e6adce80d2d034c31b67cfb, NAME => 'TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T08:59:35,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 1c52b94a9e6adce80d2d034c31b67cfb, NAME => 'TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T08:59:35,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1c52b94a9e6adce80d2d034c31b67cfb, NAME => 'TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1c52b94a9e6adce80d2d034c31b67cfb, NAME => 'TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T08:59:35,340 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:35,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43628 deadline: 1732093235337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:35,341 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:35,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43672 deadline: 1732093235337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:35,341 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to be1eb2620e0e,34223,1732093159960 for 'test_row_0' in 1c52b94a9e6adce80d2d034c31b67cfb of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4202 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-20T08:59:35,342 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:35,342 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:35,342 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:35,343 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to be1eb2620e0e,34223,1732093159960 for 'test_row_2' in 1c52b94a9e6adce80d2d034c31b67cfb of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4199 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-20T08:59:35,343 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:35,343 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:35,343 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:35,350 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:35,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43620 deadline: 1732093235347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:35,351 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to be1eb2620e0e,34223,1732093159960 for 'test_row_2' in 1c52b94a9e6adce80d2d034c31b67cfb of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4210 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-20T08:59:35,352 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:35,352 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:35,352 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:35,378 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-20T08:59:35,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb. 2024-11-20T08:59:35,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb. as already flushing 2024-11-20T08:59:35,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb. 2024-11-20T08:59:35,378 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 1c52b94a9e6adce80d2d034c31b67cfb, NAME => 'TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T08:59:35,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 1c52b94a9e6adce80d2d034c31b67cfb, NAME => 'TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T08:59:35,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1c52b94a9e6adce80d2d034c31b67cfb, NAME => 'TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1c52b94a9e6adce80d2d034c31b67cfb, NAME => 'TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T08:59:35,388 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:35,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43608 deadline: 1732093235387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:35,388 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:35,388 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:35,389 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:35,391 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:35,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43644 deadline: 1732093235389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:35,392 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:35,392 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:35,392 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:35,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-20T08:59:35,503 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=320 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/C/48e572da1fb345e1b8ac33095a43d011 2024-11-20T08:59:35,511 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/A/7ab3be557dbf4ae0a380a5d28adf2d18 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/7ab3be557dbf4ae0a380a5d28adf2d18 2024-11-20T08:59:35,518 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/7ab3be557dbf4ae0a380a5d28adf2d18, entries=150, sequenceid=320, filesize=12.0 K 2024-11-20T08:59:35,521 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/B/8c563d2983a34d87a1dcc4db311791a6 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/8c563d2983a34d87a1dcc4db311791a6 2024-11-20T08:59:35,527 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/8c563d2983a34d87a1dcc4db311791a6, entries=150, sequenceid=320, filesize=12.0 K 2024-11-20T08:59:35,533 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/C/48e572da1fb345e1b8ac33095a43d011 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/C/48e572da1fb345e1b8ac33095a43d011 2024-11-20T08:59:35,534 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-20T08:59:35,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb. 2024-11-20T08:59:35,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb. as already flushing 2024-11-20T08:59:35,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb. 2024-11-20T08:59:35,536 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 1c52b94a9e6adce80d2d034c31b67cfb, NAME => 'TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T08:59:35,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 1c52b94a9e6adce80d2d034c31b67cfb, NAME => 'TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T08:59:35,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1c52b94a9e6adce80d2d034c31b67cfb, NAME => 'TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1c52b94a9e6adce80d2d034c31b67cfb, NAME => 'TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T08:59:35,554 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/C/48e572da1fb345e1b8ac33095a43d011, entries=150, sequenceid=320, filesize=12.0 K 2024-11-20T08:59:35,556 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=53.67 KB/54960 for 1c52b94a9e6adce80d2d034c31b67cfb in 1332ms, sequenceid=320, compaction requested=false 2024-11-20T08:59:35,556 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1c52b94a9e6adce80d2d034c31b67cfb: 2024-11-20T08:59:35,691 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-20T08:59:35,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb. 2024-11-20T08:59:35,692 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing 1c52b94a9e6adce80d2d034c31b67cfb 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-20T08:59:35,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/A/52b72b396afc4518adee8690be75d91b is 50, key is test_row_0/A:col10/1732093174246/Put/seqid=0 2024-11-20T08:59:35,712 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073741902_1078 (size=12301) 2024-11-20T08:59:36,113 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=331 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/A/52b72b396afc4518adee8690be75d91b 2024-11-20T08:59:36,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/B/e183aa0a2891486caf4578dc9a002b49 is 50, key is test_row_0/B:col10/1732093174246/Put/seqid=0 2024-11-20T08:59:36,159 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073741903_1079 (size=12301) 2024-11-20T08:59:36,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(8855): Flush requested on 1c52b94a9e6adce80d2d034c31b67cfb 2024-11-20T08:59:36,399 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb. as already flushing 2024-11-20T08:59:36,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-20T08:59:36,483 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:36,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43608 deadline: 1732093236478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:36,484 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:36,484 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:36,484 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:36,486 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:36,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43644 deadline: 1732093236483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:36,486 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:36,487 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:36,487 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:36,562 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=331 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/B/e183aa0a2891486caf4578dc9a002b49 2024-11-20T08:59:36,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/C/18586f39ed8440f29b66f6e7ee1bc1b2 is 50, key is test_row_0/C:col10/1732093174246/Put/seqid=0 2024-11-20T08:59:36,590 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073741904_1080 (size=12301) 2024-11-20T08:59:36,591 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=331 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/C/18586f39ed8440f29b66f6e7ee1bc1b2 2024-11-20T08:59:36,591 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:36,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43608 deadline: 1732093236587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:36,592 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:36,592 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:36,592 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:36,599 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:36,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43644 deadline: 1732093236597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:36,599 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:36,599 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:36,599 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:36,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/A/52b72b396afc4518adee8690be75d91b as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/52b72b396afc4518adee8690be75d91b 2024-11-20T08:59:36,607 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/52b72b396afc4518adee8690be75d91b, entries=150, sequenceid=331, filesize=12.0 K 2024-11-20T08:59:36,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/B/e183aa0a2891486caf4578dc9a002b49 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/e183aa0a2891486caf4578dc9a002b49 2024-11-20T08:59:36,615 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/e183aa0a2891486caf4578dc9a002b49, entries=150, sequenceid=331, filesize=12.0 K 2024-11-20T08:59:36,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/C/18586f39ed8440f29b66f6e7ee1bc1b2 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/C/18586f39ed8440f29b66f6e7ee1bc1b2 2024-11-20T08:59:36,624 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/C/18586f39ed8440f29b66f6e7ee1bc1b2, entries=150, sequenceid=331, filesize=12.0 K 2024-11-20T08:59:36,625 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 1c52b94a9e6adce80d2d034c31b67cfb in 933ms, sequenceid=331, compaction requested=true 2024-11-20T08:59:36,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for 1c52b94a9e6adce80d2d034c31b67cfb: 2024-11-20T08:59:36,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb. 2024-11-20T08:59:36,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-11-20T08:59:36,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-11-20T08:59:36,630 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-20T08:59:36,630 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3330 sec 2024-11-20T08:59:36,634 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees in 2.3440 sec 2024-11-20T08:59:36,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(8855): Flush requested on 1c52b94a9e6adce80d2d034c31b67cfb 2024-11-20T08:59:36,800 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T08:59:36,800 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1c52b94a9e6adce80d2d034c31b67cfb 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-20T08:59:36,814 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/A/e69ecd2769af47a2b1bffe0c9d411e87 is 50, key is test_row_0/A:col10/1732093176469/Put/seqid=0 2024-11-20T08:59:36,827 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:36,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43608 deadline: 1732093236822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:36,828 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:36,828 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:36,828 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:36,828 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:36,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43644 deadline: 1732093236823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:36,828 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073741905_1081 (size=12301) 2024-11-20T08:59:36,828 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:36,829 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:36,829 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:36,829 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=357 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/A/e69ecd2769af47a2b1bffe0c9d411e87 2024-11-20T08:59:36,841 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/B/51816aaecd03437bb614ac0fd7e0703e is 50, key is test_row_0/B:col10/1732093176469/Put/seqid=0 2024-11-20T08:59:36,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073741906_1082 (size=12301) 2024-11-20T08:59:36,939 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:36,940 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:36,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43608 deadline: 1732093236937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:36,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43644 deadline: 1732093236938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:36,940 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:36,940 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:36,940 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:36,940 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:36,941 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:36,941 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:37,154 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:37,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43608 deadline: 1732093237149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:37,154 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:37,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43644 deadline: 1732093237149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:37,155 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:37,155 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:37,155 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:37,155 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:37,155 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:37,155 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:37,255 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=357 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/B/51816aaecd03437bb614ac0fd7e0703e 2024-11-20T08:59:37,273 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/C/c3426d8981c94354b5e8c062bc6bf12a is 50, key is test_row_0/C:col10/1732093176469/Put/seqid=0 2024-11-20T08:59:37,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073741907_1083 (size=12301) 2024-11-20T08:59:37,309 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=357 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/C/c3426d8981c94354b5e8c062bc6bf12a 2024-11-20T08:59:37,317 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/A/e69ecd2769af47a2b1bffe0c9d411e87 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/e69ecd2769af47a2b1bffe0c9d411e87 2024-11-20T08:59:37,323 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/e69ecd2769af47a2b1bffe0c9d411e87, entries=150, sequenceid=357, filesize=12.0 K 2024-11-20T08:59:37,324 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/B/51816aaecd03437bb614ac0fd7e0703e as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/51816aaecd03437bb614ac0fd7e0703e 2024-11-20T08:59:37,332 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/51816aaecd03437bb614ac0fd7e0703e, entries=150, sequenceid=357, filesize=12.0 K 2024-11-20T08:59:37,333 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/C/c3426d8981c94354b5e8c062bc6bf12a as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/C/c3426d8981c94354b5e8c062bc6bf12a 2024-11-20T08:59:37,347 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/C/c3426d8981c94354b5e8c062bc6bf12a, entries=150, sequenceid=357, filesize=12.0 K 2024-11-20T08:59:37,350 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 1c52b94a9e6adce80d2d034c31b67cfb in 549ms, sequenceid=357, compaction requested=true 2024-11-20T08:59:37,350 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1c52b94a9e6adce80d2d034c31b67cfb: 2024-11-20T08:59:37,350 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1c52b94a9e6adce80d2d034c31b67cfb:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T08:59:37,350 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T08:59:37,350 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T08:59:37,350 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T08:59:37,350 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1c52b94a9e6adce80d2d034c31b67cfb:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T08:59:37,364 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T08:59:37,365 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1c52b94a9e6adce80d2d034c31b67cfb:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T08:59:37,365 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T08:59:37,365 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49886 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T08:59:37,365 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HStore(1541): 1c52b94a9e6adce80d2d034c31b67cfb/A is initiating minor compaction (all files) 2024-11-20T08:59:37,365 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49886 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T08:59:37,365 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1c52b94a9e6adce80d2d034c31b67cfb/A in TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb. 2024-11-20T08:59:37,365 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1541): 1c52b94a9e6adce80d2d034c31b67cfb/B is initiating minor compaction (all files) 2024-11-20T08:59:37,365 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1c52b94a9e6adce80d2d034c31b67cfb/B in TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb. 2024-11-20T08:59:37,365 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/9fcdd67484d546e2ae02bbbacc79851b, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/7ab3be557dbf4ae0a380a5d28adf2d18, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/52b72b396afc4518adee8690be75d91b, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/e69ecd2769af47a2b1bffe0c9d411e87] into tmpdir=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp, totalSize=48.7 K 2024-11-20T08:59:37,366 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/80982404f8a746e496f91705020a9fed, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/8c563d2983a34d87a1dcc4db311791a6, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/e183aa0a2891486caf4578dc9a002b49, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/51816aaecd03437bb614ac0fd7e0703e] into tmpdir=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp, totalSize=48.7 K 2024-11-20T08:59:37,366 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting 80982404f8a746e496f91705020a9fed, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1732093172944 2024-11-20T08:59:37,366 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9fcdd67484d546e2ae02bbbacc79851b, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1732093172944 2024-11-20T08:59:37,367 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting 8c563d2983a34d87a1dcc4db311791a6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1732093173064 2024-11-20T08:59:37,367 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7ab3be557dbf4ae0a380a5d28adf2d18, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1732093173064 2024-11-20T08:59:37,367 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting e183aa0a2891486caf4578dc9a002b49, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=331, earliestPutTs=1732093174239 2024-11-20T08:59:37,368 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.Compactor(225): Compacting 52b72b396afc4518adee8690be75d91b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=331, earliestPutTs=1732093174239 2024-11-20T08:59:37,368 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting 51816aaecd03437bb614ac0fd7e0703e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=357, earliestPutTs=1732093176469 2024-11-20T08:59:37,368 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.Compactor(225): Compacting e69ecd2769af47a2b1bffe0c9d411e87, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=357, earliestPutTs=1732093176469 2024-11-20T08:59:37,387 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1c52b94a9e6adce80d2d034c31b67cfb#B#compaction#71 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-20T08:59:37,387 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1c52b94a9e6adce80d2d034c31b67cfb#A#compaction#72 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T08:59:37,388 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/A/c958825b719a4f51a7e7fb3646106d95 is 50, key is test_row_0/A:col10/1732093176469/Put/seqid=0 2024-11-20T08:59:37,391 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/B/3cc6bc716fad4c2fbda70bb12dd2920f is 50, key is test_row_0/B:col10/1732093176469/Put/seqid=0 2024-11-20T08:59:37,404 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073741908_1084 (size=13119) 2024-11-20T08:59:37,424 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/A/c958825b719a4f51a7e7fb3646106d95 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/c958825b719a4f51a7e7fb3646106d95 2024-11-20T08:59:37,431 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1c52b94a9e6adce80d2d034c31b67cfb/A of 1c52b94a9e6adce80d2d034c31b67cfb into c958825b719a4f51a7e7fb3646106d95(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T08:59:37,432 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1c52b94a9e6adce80d2d034c31b67cfb: 2024-11-20T08:59:37,432 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., storeName=1c52b94a9e6adce80d2d034c31b67cfb/A, priority=12, startTime=1732093177350; duration=0sec 2024-11-20T08:59:37,432 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T08:59:37,432 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1c52b94a9e6adce80d2d034c31b67cfb:A 2024-11-20T08:59:37,432 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T08:59:37,434 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49852 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T08:59:37,434 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HStore(1541): 1c52b94a9e6adce80d2d034c31b67cfb/C is initiating minor compaction (all files) 2024-11-20T08:59:37,434 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1c52b94a9e6adce80d2d034c31b67cfb/C in TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb. 2024-11-20T08:59:37,434 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/C/c586a4522ac94f06a72e6ce750175160, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/C/48e572da1fb345e1b8ac33095a43d011, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/C/18586f39ed8440f29b66f6e7ee1bc1b2, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/C/c3426d8981c94354b5e8c062bc6bf12a] into tmpdir=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp, totalSize=48.7 K 2024-11-20T08:59:37,435 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.Compactor(225): Compacting c586a4522ac94f06a72e6ce750175160, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1732093172944 2024-11-20T08:59:37,435 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.Compactor(225): Compacting 48e572da1fb345e1b8ac33095a43d011, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1732093173064 2024-11-20T08:59:37,436 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.Compactor(225): Compacting 18586f39ed8440f29b66f6e7ee1bc1b2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=331, earliestPutTs=1732093174239 2024-11-20T08:59:37,436 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.Compactor(225): Compacting c3426d8981c94354b5e8c062bc6bf12a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=357, earliestPutTs=1732093176469 2024-11-20T08:59:37,446 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073741909_1085 (size=13119) 2024-11-20T08:59:37,456 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/B/3cc6bc716fad4c2fbda70bb12dd2920f as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/3cc6bc716fad4c2fbda70bb12dd2920f 2024-11-20T08:59:37,459 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1c52b94a9e6adce80d2d034c31b67cfb#C#compaction#73 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T08:59:37,460 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/C/00759053465245ca936c4cea0d248e6f is 50, key is test_row_0/C:col10/1732093176469/Put/seqid=0 2024-11-20T08:59:37,469 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1c52b94a9e6adce80d2d034c31b67cfb/B of 1c52b94a9e6adce80d2d034c31b67cfb into 3cc6bc716fad4c2fbda70bb12dd2920f(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T08:59:37,470 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1c52b94a9e6adce80d2d034c31b67cfb: 2024-11-20T08:59:37,470 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., storeName=1c52b94a9e6adce80d2d034c31b67cfb/B, priority=12, startTime=1732093177350; duration=0sec 2024-11-20T08:59:37,470 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T08:59:37,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073741910_1086 (size=13085) 2024-11-20T08:59:37,470 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1c52b94a9e6adce80d2d034c31b67cfb:B 2024-11-20T08:59:37,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(8855): Flush requested on 1c52b94a9e6adce80d2d034c31b67cfb 2024-11-20T08:59:37,471 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T08:59:37,471 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1c52b94a9e6adce80d2d034c31b67cfb 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-20T08:59:37,482 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/A/66966ad6341e413e9e533ab0122d55cb is 50, key is test_row_0/A:col10/1732093176821/Put/seqid=0 2024-11-20T08:59:37,485 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/C/00759053465245ca936c4cea0d248e6f as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/C/00759053465245ca936c4cea0d248e6f 2024-11-20T08:59:37,500 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073741911_1087 (size=17181) 2024-11-20T08:59:37,500 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1c52b94a9e6adce80d2d034c31b67cfb/C of 1c52b94a9e6adce80d2d034c31b67cfb into 00759053465245ca936c4cea0d248e6f(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T08:59:37,500 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1c52b94a9e6adce80d2d034c31b67cfb: 2024-11-20T08:59:37,501 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., storeName=1c52b94a9e6adce80d2d034c31b67cfb/C, priority=12, startTime=1732093177365; duration=0sec 2024-11-20T08:59:37,501 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T08:59:37,501 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1c52b94a9e6adce80d2d034c31b67cfb:C 2024-11-20T08:59:37,502 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=371 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/A/66966ad6341e413e9e533ab0122d55cb 2024-11-20T08:59:37,522 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/B/c73a991c58c8427685ee4775e0024d86 is 50, key is test_row_0/B:col10/1732093176821/Put/seqid=0 2024-11-20T08:59:37,545 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073741912_1088 (size=12301) 2024-11-20T08:59:37,546 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=371 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/B/c73a991c58c8427685ee4775e0024d86 2024-11-20T08:59:37,549 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:37,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43644 deadline: 1732093237545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:37,550 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:37,550 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:37,550 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:37,550 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:37,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43608 deadline: 1732093237547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:37,551 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:37,551 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:37,551 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:37,556 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/C/62baecee9b0f4cb1acdb5ba5d32913c5 is 50, key is test_row_0/C:col10/1732093176821/Put/seqid=0 2024-11-20T08:59:37,566 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073741913_1089 (size=12301) 2024-11-20T08:59:37,567 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=371 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/C/62baecee9b0f4cb1acdb5ba5d32913c5 2024-11-20T08:59:37,573 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/A/66966ad6341e413e9e533ab0122d55cb as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/66966ad6341e413e9e533ab0122d55cb 2024-11-20T08:59:37,580 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/66966ad6341e413e9e533ab0122d55cb, entries=250, sequenceid=371, filesize=16.8 K 2024-11-20T08:59:37,582 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/B/c73a991c58c8427685ee4775e0024d86 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/c73a991c58c8427685ee4775e0024d86 2024-11-20T08:59:37,592 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/c73a991c58c8427685ee4775e0024d86, entries=150, sequenceid=371, filesize=12.0 K 2024-11-20T08:59:37,594 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/C/62baecee9b0f4cb1acdb5ba5d32913c5 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/C/62baecee9b0f4cb1acdb5ba5d32913c5 2024-11-20T08:59:37,600 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/C/62baecee9b0f4cb1acdb5ba5d32913c5, entries=150, sequenceid=371, filesize=12.0 K 2024-11-20T08:59:37,603 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 1c52b94a9e6adce80d2d034c31b67cfb in 132ms, sequenceid=371, compaction requested=false 2024-11-20T08:59:37,603 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1c52b94a9e6adce80d2d034c31b67cfb: 2024-11-20T08:59:37,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(8855): Flush requested on 1c52b94a9e6adce80d2d034c31b67cfb 2024-11-20T08:59:37,662 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T08:59:37,662 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1c52b94a9e6adce80d2d034c31b67cfb 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-20T08:59:37,672 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/A/f9fafbb9618c4117977726aa07854ff6 is 50, key is test_row_0/A:col10/1732093177661/Put/seqid=0 2024-11-20T08:59:37,693 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073741914_1090 (size=14741) 2024-11-20T08:59:37,695 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:37,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43608 deadline: 1732093237691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:37,696 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:37,697 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:37,697 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:37,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43644 deadline: 1732093237691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:37,697 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:37,697 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:37,697 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:37,697 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:37,810 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:37,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43644 deadline: 1732093237807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:37,811 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:37,811 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:37,811 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:37,811 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:37,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43608 deadline: 1732093237807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:37,812 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:37,812 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:37,812 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:38,019 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:38,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43644 deadline: 1732093238017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:38,020 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:38,020 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:38,020 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:38,020 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:38,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43608 deadline: 1732093238017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:38,021 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:38,021 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:38,021 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:38,095 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=398 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/A/f9fafbb9618c4117977726aa07854ff6 2024-11-20T08:59:38,109 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/B/f0e64b206b834513bf0d39333ca33e94 is 50, key is test_row_0/B:col10/1732093177661/Put/seqid=0 2024-11-20T08:59:38,116 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073741915_1091 (size=12301) 2024-11-20T08:59:38,329 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:38,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43644 deadline: 1732093238327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:38,330 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:38,330 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:38,330 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:38,330 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:38,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43608 deadline: 1732093238327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:38,331 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:38,331 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:38,331 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:38,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-20T08:59:38,447 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T08:59:38,448 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T08:59:38,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] procedure2.ProcedureExecutor(1139): Stored pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees 2024-11-20T08:59:38,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-20T08:59:38,450 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T08:59:38,451 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T08:59:38,451 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T08:59:38,517 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=398 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/B/f0e64b206b834513bf0d39333ca33e94 2024-11-20T08:59:38,527 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/C/e60b34c184bd43a4b819aa3c0a60cd1c is 50, key is test_row_0/C:col10/1732093177661/Put/seqid=0 2024-11-20T08:59:38,531 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073741916_1092 (size=12301) 2024-11-20T08:59:38,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-20T08:59:38,604 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-20T08:59:38,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb. 2024-11-20T08:59:38,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb. as already flushing 2024-11-20T08:59:38,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb. 2024-11-20T08:59:38,605 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 1c52b94a9e6adce80d2d034c31b67cfb, NAME => 'TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T08:59:38,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 1c52b94a9e6adce80d2d034c31b67cfb, NAME => 'TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T08:59:38,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1c52b94a9e6adce80d2d034c31b67cfb, NAME => 'TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1c52b94a9e6adce80d2d034c31b67cfb, NAME => 'TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T08:59:38,759 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-20T08:59:38,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb. 2024-11-20T08:59:38,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb. as already flushing 2024-11-20T08:59:38,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb. 2024-11-20T08:59:38,760 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 1c52b94a9e6adce80d2d034c31b67cfb, NAME => 'TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T08:59:38,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 1c52b94a9e6adce80d2d034c31b67cfb, NAME => 'TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T08:59:38,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1c52b94a9e6adce80d2d034c31b67cfb, NAME => 'TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1c52b94a9e6adce80d2d034c31b67cfb, NAME => 'TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T08:59:38,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-20T08:59:38,841 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:38,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43644 deadline: 1732093238837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:38,842 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:38,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43608 deadline: 1732093238837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:38,843 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:38,843 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:38,843 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:38,844 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:38,844 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:38,844 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:38,917 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-20T08:59:38,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb. 2024-11-20T08:59:38,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb. as already flushing 2024-11-20T08:59:38,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb. 2024-11-20T08:59:38,917 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 1c52b94a9e6adce80d2d034c31b67cfb, NAME => 'TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T08:59:38,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 1c52b94a9e6adce80d2d034c31b67cfb, NAME => 'TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T08:59:38,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1c52b94a9e6adce80d2d034c31b67cfb, NAME => 'TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1c52b94a9e6adce80d2d034c31b67cfb, NAME => 'TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T08:59:38,933 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=398 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/C/e60b34c184bd43a4b819aa3c0a60cd1c 2024-11-20T08:59:38,944 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/A/f9fafbb9618c4117977726aa07854ff6 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/f9fafbb9618c4117977726aa07854ff6 2024-11-20T08:59:38,956 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/f9fafbb9618c4117977726aa07854ff6, entries=200, sequenceid=398, filesize=14.4 K 2024-11-20T08:59:38,957 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/B/f0e64b206b834513bf0d39333ca33e94 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/f0e64b206b834513bf0d39333ca33e94 2024-11-20T08:59:38,975 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/f0e64b206b834513bf0d39333ca33e94, entries=150, sequenceid=398, filesize=12.0 K 2024-11-20T08:59:38,976 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/C/e60b34c184bd43a4b819aa3c0a60cd1c as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/C/e60b34c184bd43a4b819aa3c0a60cd1c 2024-11-20T08:59:38,984 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/C/e60b34c184bd43a4b819aa3c0a60cd1c, entries=150, sequenceid=398, filesize=12.0 K 2024-11-20T08:59:38,986 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=53.67 KB/54960 for 1c52b94a9e6adce80d2d034c31b67cfb in 1323ms, sequenceid=398, compaction requested=true 2024-11-20T08:59:38,986 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1c52b94a9e6adce80d2d034c31b67cfb: 2024-11-20T08:59:38,986 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1c52b94a9e6adce80d2d034c31b67cfb:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T08:59:38,986 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T08:59:38,986 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T08:59:38,986 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T08:59:38,987 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1c52b94a9e6adce80d2d034c31b67cfb:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T08:59:38,987 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T08:59:38,987 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1c52b94a9e6adce80d2d034c31b67cfb:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T08:59:38,987 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T08:59:38,989 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T08:59:38,989 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1541): 1c52b94a9e6adce80d2d034c31b67cfb/B is initiating minor compaction (all files) 2024-11-20T08:59:38,989 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1c52b94a9e6adce80d2d034c31b67cfb/B in TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb. 2024-11-20T08:59:38,989 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/3cc6bc716fad4c2fbda70bb12dd2920f, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/c73a991c58c8427685ee4775e0024d86, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/f0e64b206b834513bf0d39333ca33e94] into tmpdir=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp, totalSize=36.8 K 2024-11-20T08:59:38,990 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 45041 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T08:59:38,990 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HStore(1541): 1c52b94a9e6adce80d2d034c31b67cfb/A is initiating minor compaction (all files) 2024-11-20T08:59:38,990 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1c52b94a9e6adce80d2d034c31b67cfb/A in TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb. 2024-11-20T08:59:38,990 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/c958825b719a4f51a7e7fb3646106d95, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/66966ad6341e413e9e533ab0122d55cb, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/f9fafbb9618c4117977726aa07854ff6] into tmpdir=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp, totalSize=44.0 K 2024-11-20T08:59:38,990 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting 3cc6bc716fad4c2fbda70bb12dd2920f, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=357, earliestPutTs=1732093176469 2024-11-20T08:59:38,991 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.Compactor(225): Compacting c958825b719a4f51a7e7fb3646106d95, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=357, earliestPutTs=1732093176469 2024-11-20T08:59:38,991 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.Compactor(225): Compacting 66966ad6341e413e9e533ab0122d55cb, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=371, earliestPutTs=1732093176808 2024-11-20T08:59:38,991 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting c73a991c58c8427685ee4775e0024d86, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=371, earliestPutTs=1732093176808 2024-11-20T08:59:38,992 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.Compactor(225): Compacting f9fafbb9618c4117977726aa07854ff6, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=398, earliestPutTs=1732093177533 2024-11-20T08:59:38,992 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting f0e64b206b834513bf0d39333ca33e94, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=398, earliestPutTs=1732093177545 2024-11-20T08:59:39,009 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1c52b94a9e6adce80d2d034c31b67cfb#A#compaction#80 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T08:59:39,010 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/A/4662eb873e1448c78f2efb17a466b9af is 50, key is test_row_0/A:col10/1732093177661/Put/seqid=0 2024-11-20T08:59:39,018 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1c52b94a9e6adce80d2d034c31b67cfb#B#compaction#81 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T08:59:39,019 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/B/9582a0ce87a041249781a7dfd8a08d47 is 50, key is test_row_0/B:col10/1732093177661/Put/seqid=0 2024-11-20T08:59:39,022 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073741917_1093 (size=13221) 2024-11-20T08:59:39,033 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073741918_1094 (size=13221) 2024-11-20T08:59:39,074 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-20T08:59:39,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb. 2024-11-20T08:59:39,075 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2902): Flushing 1c52b94a9e6adce80d2d034c31b67cfb 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-20T08:59:39,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-20T08:59:39,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/A/f1ea035273134700b0f3bfbb548bd814 is 50, key is test_row_0/A:col10/1732093177689/Put/seqid=0 2024-11-20T08:59:39,178 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073741919_1095 (size=9857) 2024-11-20T08:59:39,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(8855): Flush requested on 1c52b94a9e6adce80d2d034c31b67cfb 2024-11-20T08:59:39,385 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb. as already flushing 2024-11-20T08:59:39,441 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/A/4662eb873e1448c78f2efb17a466b9af as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/4662eb873e1448c78f2efb17a466b9af 2024-11-20T08:59:39,449 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/B/9582a0ce87a041249781a7dfd8a08d47 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/9582a0ce87a041249781a7dfd8a08d47 2024-11-20T08:59:39,452 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1c52b94a9e6adce80d2d034c31b67cfb/A of 1c52b94a9e6adce80d2d034c31b67cfb into 4662eb873e1448c78f2efb17a466b9af(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T08:59:39,453 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1c52b94a9e6adce80d2d034c31b67cfb: 2024-11-20T08:59:39,453 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., storeName=1c52b94a9e6adce80d2d034c31b67cfb/A, priority=13, startTime=1732093178986; duration=0sec 2024-11-20T08:59:39,453 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T08:59:39,453 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1c52b94a9e6adce80d2d034c31b67cfb:A 2024-11-20T08:59:39,453 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T08:59:39,455 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T08:59:39,456 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HStore(1541): 1c52b94a9e6adce80d2d034c31b67cfb/C is initiating minor compaction (all files) 2024-11-20T08:59:39,456 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1c52b94a9e6adce80d2d034c31b67cfb/C in TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb. 2024-11-20T08:59:39,456 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/C/00759053465245ca936c4cea0d248e6f, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/C/62baecee9b0f4cb1acdb5ba5d32913c5, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/C/e60b34c184bd43a4b819aa3c0a60cd1c] into tmpdir=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp, totalSize=36.8 K 2024-11-20T08:59:39,457 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.Compactor(225): Compacting 00759053465245ca936c4cea0d248e6f, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=357, earliestPutTs=1732093176469 2024-11-20T08:59:39,458 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.Compactor(225): Compacting 62baecee9b0f4cb1acdb5ba5d32913c5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=371, earliestPutTs=1732093176808 2024-11-20T08:59:39,459 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.Compactor(225): Compacting e60b34c184bd43a4b819aa3c0a60cd1c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=398, earliestPutTs=1732093177545 2024-11-20T08:59:39,460 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1c52b94a9e6adce80d2d034c31b67cfb/B of 1c52b94a9e6adce80d2d034c31b67cfb into 9582a0ce87a041249781a7dfd8a08d47(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T08:59:39,460 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1c52b94a9e6adce80d2d034c31b67cfb: 2024-11-20T08:59:39,460 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., storeName=1c52b94a9e6adce80d2d034c31b67cfb/B, priority=13, startTime=1732093178986; duration=0sec 2024-11-20T08:59:39,460 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T08:59:39,460 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1c52b94a9e6adce80d2d034c31b67cfb:B 2024-11-20T08:59:39,465 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:39,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43620 deadline: 1732093239456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:39,466 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:39,467 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:39,467 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:39,473 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:39,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43672 deadline: 1732093239466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:39,473 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:39,474 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:39,474 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:39,474 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1c52b94a9e6adce80d2d034c31b67cfb#C#compaction#83 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T08:59:39,475 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/C/5c999896ce04463182cb30a2dabe28e5 is 50, key is test_row_0/C:col10/1732093177661/Put/seqid=0 2024-11-20T08:59:39,473 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:39,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43628 deadline: 1732093239469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:39,480 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:39,481 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:39,481 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:39,481 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073741920_1096 (size=13187) 2024-11-20T08:59:39,505 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/C/5c999896ce04463182cb30a2dabe28e5 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/C/5c999896ce04463182cb30a2dabe28e5 2024-11-20T08:59:39,521 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1c52b94a9e6adce80d2d034c31b67cfb/C of 1c52b94a9e6adce80d2d034c31b67cfb into 5c999896ce04463182cb30a2dabe28e5(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T08:59:39,521 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1c52b94a9e6adce80d2d034c31b67cfb: 2024-11-20T08:59:39,521 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., storeName=1c52b94a9e6adce80d2d034c31b67cfb/C, priority=13, startTime=1732093178987; duration=0sec 2024-11-20T08:59:39,522 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T08:59:39,522 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1c52b94a9e6adce80d2d034c31b67cfb:C 2024-11-20T08:59:39,580 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=409 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/A/f1ea035273134700b0f3bfbb548bd814 2024-11-20T08:59:39,583 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:39,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43672 deadline: 1732093239577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:39,585 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:39,585 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:39,585 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:39,586 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:39,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43620 deadline: 1732093239577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:39,587 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:39,587 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:39,587 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:39,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-20T08:59:39,589 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:39,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43628 deadline: 1732093239588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:39,589 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:39,589 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:39,589 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:39,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/B/dfc65b4a655c47a0a607806d072f14b2 is 50, key is test_row_0/B:col10/1732093177689/Put/seqid=0 2024-11-20T08:59:39,661 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073741921_1097 (size=9857) 2024-11-20T08:59:39,795 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:39,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43672 deadline: 1732093239789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:39,796 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:39,796 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:39,796 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:39,800 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:39,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43628 deadline: 1732093239798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:39,800 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:39,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43620 deadline: 1732093239798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:39,801 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:39,801 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:39,801 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:39,802 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:39,802 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:39,802 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:39,849 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:39,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43644 deadline: 1732093239847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:39,850 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to be1eb2620e0e,34223,1732093159960 for 'test_row_2' in 1c52b94a9e6adce80d2d034c31b67cfb of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2159 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-20T08:59:39,851 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:39,851 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:39,851 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:39,859 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:39,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43608 deadline: 1732093239857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:39,859 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to be1eb2620e0e,34223,1732093159960 for 'test_row_2' in 1c52b94a9e6adce80d2d034c31b67cfb of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2168 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-20T08:59:39,860 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:39,860 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:39,860 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:40,073 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=409 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/B/dfc65b4a655c47a0a607806d072f14b2 2024-11-20T08:59:40,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/C/0ae3916d365d44a09956735c983b4baa is 50, key is test_row_0/C:col10/1732093177689/Put/seqid=0 2024-11-20T08:59:40,107 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073741922_1098 (size=9857) 2024-11-20T08:59:40,108 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=409 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/C/0ae3916d365d44a09956735c983b4baa 2024-11-20T08:59:40,109 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:40,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43672 deadline: 1732093240107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:40,110 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:40,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43628 deadline: 1732093240107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:40,111 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:40,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43620 deadline: 1732093240107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:40,111 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:40,111 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:40,111 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:40,111 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:40,112 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:40,112 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:40,112 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:40,112 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:40,112 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:40,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/A/f1ea035273134700b0f3bfbb548bd814 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/f1ea035273134700b0f3bfbb548bd814 2024-11-20T08:59:40,142 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/f1ea035273134700b0f3bfbb548bd814, entries=100, sequenceid=409, filesize=9.6 K 2024-11-20T08:59:40,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/B/dfc65b4a655c47a0a607806d072f14b2 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/dfc65b4a655c47a0a607806d072f14b2 2024-11-20T08:59:40,154 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/dfc65b4a655c47a0a607806d072f14b2, entries=100, sequenceid=409, filesize=9.6 K 2024-11-20T08:59:40,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/C/0ae3916d365d44a09956735c983b4baa as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/C/0ae3916d365d44a09956735c983b4baa 2024-11-20T08:59:40,174 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/C/0ae3916d365d44a09956735c983b4baa, entries=100, sequenceid=409, filesize=9.6 K 2024-11-20T08:59:40,176 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 1c52b94a9e6adce80d2d034c31b67cfb in 1101ms, sequenceid=409, compaction requested=false 2024-11-20T08:59:40,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2603): Flush status journal for 1c52b94a9e6adce80d2d034c31b67cfb: 2024-11-20T08:59:40,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb. 2024-11-20T08:59:40,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=16 2024-11-20T08:59:40,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.HMaster(4169): Remote procedure done, pid=16 2024-11-20T08:59:40,185 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-11-20T08:59:40,185 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7280 sec 2024-11-20T08:59:40,187 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees in 1.7380 sec 2024-11-20T08:59:40,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-20T08:59:40,597 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T08:59:40,598 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T08:59:40,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees 2024-11-20T08:59:40,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-20T08:59:40,601 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T08:59:40,602 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T08:59:40,602 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T08:59:40,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(8855): Flush requested on 1c52b94a9e6adce80d2d034c31b67cfb 2024-11-20T08:59:40,620 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T08:59:40,620 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1c52b94a9e6adce80d2d034c31b67cfb 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-20T08:59:40,628 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/A/b39cf939cd114fa2aff5118b56249034 is 50, key is test_row_0/A:col10/1732093180618/Put/seqid=0 2024-11-20T08:59:40,639 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:40,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43672 deadline: 1732093240635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:40,640 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:40,640 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:40,640 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:40,640 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:40,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43620 deadline: 1732093240636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:40,644 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:40,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43628 deadline: 1732093240637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:40,644 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:40,644 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:40,644 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:40,645 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:40,644 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:40,645 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:40,647 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073741923_1099 (size=12301) 2024-11-20T08:59:40,649 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=438 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/A/b39cf939cd114fa2aff5118b56249034 2024-11-20T08:59:40,663 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/B/a295cafda18248fd93b17446413328dd is 50, key is test_row_0/B:col10/1732093180618/Put/seqid=0 2024-11-20T08:59:40,690 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073741924_1100 (size=12301) 2024-11-20T08:59:40,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-20T08:59:40,748 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:40,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43672 deadline: 1732093240747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:40,749 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:40,749 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:40,749 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:40,749 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:40,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43620 deadline: 1732093240747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:40,749 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:40,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43628 deadline: 1732093240747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:40,750 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:40,750 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:40,750 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:40,750 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:40,750 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:40,751 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:40,763 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-20T08:59:40,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb. 2024-11-20T08:59:40,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb. as already flushing 2024-11-20T08:59:40,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb. 2024-11-20T08:59:40,764 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 1c52b94a9e6adce80d2d034c31b67cfb, NAME => 'TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T08:59:40,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 1c52b94a9e6adce80d2d034c31b67cfb, NAME => 'TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T08:59:40,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1c52b94a9e6adce80d2d034c31b67cfb, NAME => 'TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1c52b94a9e6adce80d2d034c31b67cfb, NAME => 'TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T08:59:40,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-20T08:59:40,920 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-20T08:59:40,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb. 2024-11-20T08:59:40,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb. as already flushing 2024-11-20T08:59:40,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb. 2024-11-20T08:59:40,920 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 1c52b94a9e6adce80d2d034c31b67cfb, NAME => 'TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T08:59:40,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 1c52b94a9e6adce80d2d034c31b67cfb, NAME => 'TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T08:59:40,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1c52b94a9e6adce80d2d034c31b67cfb, NAME => 'TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1c52b94a9e6adce80d2d034c31b67cfb, NAME => 'TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T08:59:40,969 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:40,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43672 deadline: 1732093240963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:40,969 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:40,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43628 deadline: 1732093240964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:40,970 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:40,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43620 deadline: 1732093240964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:40,971 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:40,971 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:40,971 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:40,971 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:40,971 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:40,971 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:40,972 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:40,972 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:40,972 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:41,074 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-20T08:59:41,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb. 2024-11-20T08:59:41,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb. as already flushing 2024-11-20T08:59:41,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb. 2024-11-20T08:59:41,074 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 1c52b94a9e6adce80d2d034c31b67cfb, NAME => 'TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T08:59:41,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 1c52b94a9e6adce80d2d034c31b67cfb, NAME => 'TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T08:59:41,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1c52b94a9e6adce80d2d034c31b67cfb, NAME => 'TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1c52b94a9e6adce80d2d034c31b67cfb, NAME => 'TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T08:59:41,091 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=438 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/B/a295cafda18248fd93b17446413328dd 2024-11-20T08:59:41,105 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/C/6c9cf34eeb4249fd9b7bd4fdb0d6f5c9 is 50, key is test_row_0/C:col10/1732093180618/Put/seqid=0 2024-11-20T08:59:41,110 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073741925_1101 (size=12301) 2024-11-20T08:59:41,112 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=438 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/C/6c9cf34eeb4249fd9b7bd4fdb0d6f5c9 2024-11-20T08:59:41,119 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/A/b39cf939cd114fa2aff5118b56249034 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/b39cf939cd114fa2aff5118b56249034 2024-11-20T08:59:41,133 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/b39cf939cd114fa2aff5118b56249034, entries=150, sequenceid=438, filesize=12.0 K 2024-11-20T08:59:41,134 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/B/a295cafda18248fd93b17446413328dd as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/a295cafda18248fd93b17446413328dd 2024-11-20T08:59:41,156 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/a295cafda18248fd93b17446413328dd, entries=150, sequenceid=438, filesize=12.0 K 2024-11-20T08:59:41,158 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/C/6c9cf34eeb4249fd9b7bd4fdb0d6f5c9 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/C/6c9cf34eeb4249fd9b7bd4fdb0d6f5c9 2024-11-20T08:59:41,166 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/C/6c9cf34eeb4249fd9b7bd4fdb0d6f5c9, entries=150, sequenceid=438, filesize=12.0 K 2024-11-20T08:59:41,168 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 1c52b94a9e6adce80d2d034c31b67cfb in 547ms, sequenceid=438, compaction requested=true 2024-11-20T08:59:41,168 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1c52b94a9e6adce80d2d034c31b67cfb: 2024-11-20T08:59:41,168 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1c52b94a9e6adce80d2d034c31b67cfb:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T08:59:41,168 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T08:59:41,168 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T08:59:41,168 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1c52b94a9e6adce80d2d034c31b67cfb:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T08:59:41,168 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T08:59:41,168 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1c52b94a9e6adce80d2d034c31b67cfb:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T08:59:41,168 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-20T08:59:41,168 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T08:59:41,170 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35379 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T08:59:41,170 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HStore(1541): 1c52b94a9e6adce80d2d034c31b67cfb/A is initiating minor compaction (all files) 2024-11-20T08:59:41,170 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1c52b94a9e6adce80d2d034c31b67cfb/A in TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb. 2024-11-20T08:59:41,170 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/4662eb873e1448c78f2efb17a466b9af, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/f1ea035273134700b0f3bfbb548bd814, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/b39cf939cd114fa2aff5118b56249034] into tmpdir=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp, totalSize=34.5 K 2024-11-20T08:59:41,170 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35379 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T08:59:41,170 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1541): 1c52b94a9e6adce80d2d034c31b67cfb/B is initiating minor compaction (all files) 2024-11-20T08:59:41,170 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1c52b94a9e6adce80d2d034c31b67cfb/B in TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb. 2024-11-20T08:59:41,171 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/9582a0ce87a041249781a7dfd8a08d47, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/dfc65b4a655c47a0a607806d072f14b2, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/a295cafda18248fd93b17446413328dd] into tmpdir=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp, totalSize=34.5 K 2024-11-20T08:59:41,171 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4662eb873e1448c78f2efb17a466b9af, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=398, earliestPutTs=1732093177545 2024-11-20T08:59:41,171 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting 9582a0ce87a041249781a7dfd8a08d47, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=398, earliestPutTs=1732093177545 2024-11-20T08:59:41,171 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.Compactor(225): Compacting f1ea035273134700b0f3bfbb548bd814, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=409, earliestPutTs=1732093177685 2024-11-20T08:59:41,172 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting dfc65b4a655c47a0a607806d072f14b2, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=409, earliestPutTs=1732093177685 2024-11-20T08:59:41,172 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.Compactor(225): Compacting b39cf939cd114fa2aff5118b56249034, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=438, earliestPutTs=1732093179448 2024-11-20T08:59:41,172 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting a295cafda18248fd93b17446413328dd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=438, earliestPutTs=1732093179448 2024-11-20T08:59:41,185 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1c52b94a9e6adce80d2d034c31b67cfb#A#compaction#89 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T08:59:41,186 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/A/3e40b48d7d93455abc46ce2974f19148 is 50, key is test_row_0/A:col10/1732093180618/Put/seqid=0 2024-11-20T08:59:41,198 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1c52b94a9e6adce80d2d034c31b67cfb#B#compaction#90 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T08:59:41,199 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/B/3e5841ea82c54ab69aa11e2cd14fb1a9 is 50, key is test_row_0/B:col10/1732093180618/Put/seqid=0 2024-11-20T08:59:41,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-20T08:59:41,228 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-20T08:59:41,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb. 2024-11-20T08:59:41,229 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2902): Flushing 1c52b94a9e6adce80d2d034c31b67cfb 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-20T08:59:41,250 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073741926_1102 (size=13323) 2024-11-20T08:59:41,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/A/e0beb8f7442a40c48422b266e7d5df63 is 50, key is test_row_0/A:col10/1732093180636/Put/seqid=0 2024-11-20T08:59:41,252 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073741927_1103 (size=13323) 2024-11-20T08:59:41,262 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/A/3e40b48d7d93455abc46ce2974f19148 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/3e40b48d7d93455abc46ce2974f19148 2024-11-20T08:59:41,268 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/B/3e5841ea82c54ab69aa11e2cd14fb1a9 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/3e5841ea82c54ab69aa11e2cd14fb1a9 2024-11-20T08:59:41,280 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073741928_1104 (size=12301) 2024-11-20T08:59:41,281 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb. as already flushing 2024-11-20T08:59:41,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(8855): Flush requested on 1c52b94a9e6adce80d2d034c31b67cfb 2024-11-20T08:59:41,282 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1c52b94a9e6adce80d2d034c31b67cfb/A of 1c52b94a9e6adce80d2d034c31b67cfb into 3e40b48d7d93455abc46ce2974f19148(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T08:59:41,282 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1c52b94a9e6adce80d2d034c31b67cfb: 2024-11-20T08:59:41,282 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., storeName=1c52b94a9e6adce80d2d034c31b67cfb/A, priority=13, startTime=1732093181168; duration=0sec 2024-11-20T08:59:41,282 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T08:59:41,283 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1c52b94a9e6adce80d2d034c31b67cfb:A 2024-11-20T08:59:41,283 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T08:59:41,285 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35345 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T08:59:41,285 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HStore(1541): 1c52b94a9e6adce80d2d034c31b67cfb/C is initiating minor compaction (all files) 2024-11-20T08:59:41,285 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1c52b94a9e6adce80d2d034c31b67cfb/C in TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb. 2024-11-20T08:59:41,285 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/C/5c999896ce04463182cb30a2dabe28e5, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/C/0ae3916d365d44a09956735c983b4baa, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/C/6c9cf34eeb4249fd9b7bd4fdb0d6f5c9] into tmpdir=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp, totalSize=34.5 K 2024-11-20T08:59:41,285 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1c52b94a9e6adce80d2d034c31b67cfb/B of 1c52b94a9e6adce80d2d034c31b67cfb into 3e5841ea82c54ab69aa11e2cd14fb1a9(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T08:59:41,285 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1c52b94a9e6adce80d2d034c31b67cfb: 2024-11-20T08:59:41,285 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., storeName=1c52b94a9e6adce80d2d034c31b67cfb/B, priority=13, startTime=1732093181168; duration=0sec 2024-11-20T08:59:41,285 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T08:59:41,286 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1c52b94a9e6adce80d2d034c31b67cfb:B 2024-11-20T08:59:41,286 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5c999896ce04463182cb30a2dabe28e5, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=398, earliestPutTs=1732093177545 2024-11-20T08:59:41,286 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0ae3916d365d44a09956735c983b4baa, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=409, earliestPutTs=1732093177685 2024-11-20T08:59:41,287 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6c9cf34eeb4249fd9b7bd4fdb0d6f5c9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=438, earliestPutTs=1732093179448 2024-11-20T08:59:41,287 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=448 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/A/e0beb8f7442a40c48422b266e7d5df63 2024-11-20T08:59:41,305 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1c52b94a9e6adce80d2d034c31b67cfb#C#compaction#92 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T08:59:41,306 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/C/4b004e038ded48e585be6e88fd09a463 is 50, key is test_row_0/C:col10/1732093180618/Put/seqid=0 2024-11-20T08:59:41,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/B/51f3985067d2450eb0e0f202b6395189 is 50, key is test_row_0/B:col10/1732093180636/Put/seqid=0 2024-11-20T08:59:41,335 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073741930_1106 (size=12301) 2024-11-20T08:59:41,336 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=448 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/B/51f3985067d2450eb0e0f202b6395189 2024-11-20T08:59:41,336 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073741929_1105 (size=13289) 2024-11-20T08:59:41,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/C/00aeea979ff341178675a0078c5616c2 is 50, key is test_row_0/C:col10/1732093180636/Put/seqid=0 2024-11-20T08:59:41,363 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/C/4b004e038ded48e585be6e88fd09a463 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/C/4b004e038ded48e585be6e88fd09a463 2024-11-20T08:59:41,375 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1c52b94a9e6adce80d2d034c31b67cfb/C of 1c52b94a9e6adce80d2d034c31b67cfb into 4b004e038ded48e585be6e88fd09a463(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T08:59:41,375 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1c52b94a9e6adce80d2d034c31b67cfb: 2024-11-20T08:59:41,375 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., storeName=1c52b94a9e6adce80d2d034c31b67cfb/C, priority=13, startTime=1732093181168; duration=0sec 2024-11-20T08:59:41,375 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T08:59:41,375 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1c52b94a9e6adce80d2d034c31b67cfb:C 2024-11-20T08:59:41,377 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073741931_1107 (size=12301) 2024-11-20T08:59:41,378 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=448 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/C/00aeea979ff341178675a0078c5616c2 2024-11-20T08:59:41,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/A/e0beb8f7442a40c48422b266e7d5df63 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/e0beb8f7442a40c48422b266e7d5df63 2024-11-20T08:59:41,397 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/e0beb8f7442a40c48422b266e7d5df63, entries=150, sequenceid=448, filesize=12.0 K 2024-11-20T08:59:41,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/B/51f3985067d2450eb0e0f202b6395189 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/51f3985067d2450eb0e0f202b6395189 2024-11-20T08:59:41,403 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:41,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43672 deadline: 1732093241397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:41,404 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:41,404 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:41,404 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:41,406 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/51f3985067d2450eb0e0f202b6395189, entries=150, sequenceid=448, filesize=12.0 K 2024-11-20T08:59:41,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/C/00aeea979ff341178675a0078c5616c2 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/C/00aeea979ff341178675a0078c5616c2 2024-11-20T08:59:41,412 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/C/00aeea979ff341178675a0078c5616c2, entries=150, sequenceid=448, filesize=12.0 K 2024-11-20T08:59:41,414 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=161.02 KB/164880 for 1c52b94a9e6adce80d2d034c31b67cfb in 186ms, sequenceid=448, compaction requested=false 2024-11-20T08:59:41,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2603): Flush status journal for 1c52b94a9e6adce80d2d034c31b67cfb: 2024-11-20T08:59:41,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb. 2024-11-20T08:59:41,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=18 2024-11-20T08:59:41,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.HMaster(4169): Remote procedure done, pid=18 2024-11-20T08:59:41,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(8855): Flush requested on 1c52b94a9e6adce80d2d034c31b67cfb 2024-11-20T08:59:41,419 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T08:59:41,419 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1c52b94a9e6adce80d2d034c31b67cfb 3/3 column families, dataSize=167.72 KB heapSize=440.20 KB 2024-11-20T08:59:41,425 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-11-20T08:59:41,425 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 814 msec 2024-11-20T08:59:41,428 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees in 828 msec 2024-11-20T08:59:41,460 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/A/4f6e24e4ae594f2991ae0ecfcaba82d4 is 50, key is test_row_0/A:col10/1732093181400/Put/seqid=0 2024-11-20T08:59:41,478 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:41,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43628 deadline: 1732093241472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:41,479 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:41,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43620 deadline: 1732093241472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:41,479 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:41,479 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:41,479 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:41,480 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:41,480 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:41,480 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:41,487 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073741932_1108 (size=12301) 2024-11-20T08:59:41,508 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:41,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43672 deadline: 1732093241507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:41,509 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:41,509 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:41,509 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:41,593 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:41,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43628 deadline: 1732093241587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:41,594 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:41,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43620 deadline: 1732093241587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:41,595 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:41,595 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:41,595 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:41,595 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:41,595 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:41,595 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:41,717 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:41,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43672 deadline: 1732093241717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:41,718 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:41,718 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:41,718 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:41,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-20T08:59:41,737 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T08:59:41,738 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T08:59:41,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees 2024-11-20T08:59:41,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-20T08:59:41,740 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T08:59:41,741 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T08:59:41,741 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T08:59:41,800 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:41,800 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:41,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43620 deadline: 1732093241797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:41,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43628 deadline: 1732093241797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:41,801 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:41,801 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:41,801 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:41,801 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:41,801 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:41,802 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:41,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-20T08:59:41,879 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:41,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43644 deadline: 1732093241877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:41,880 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to be1eb2620e0e,34223,1732093159960 for 'test_row_2' in 1c52b94a9e6adce80d2d034c31b67cfb of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4189 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-20T08:59:41,881 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:41,881 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:41,881 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:41,888 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=479 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/A/4f6e24e4ae594f2991ae0ecfcaba82d4 2024-11-20T08:59:41,891 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:41,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43608 deadline: 1732093241887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:41,893 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to be1eb2620e0e,34223,1732093159960 for 'test_row_2' in 1c52b94a9e6adce80d2d034c31b67cfb of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4201 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-20T08:59:41,893 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:41,893 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:41,893 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:41,894 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-20T08:59:41,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb. 2024-11-20T08:59:41,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb. as already flushing 2024-11-20T08:59:41,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb. 2024-11-20T08:59:41,894 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 1c52b94a9e6adce80d2d034c31b67cfb, NAME => 'TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T08:59:41,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 1c52b94a9e6adce80d2d034c31b67cfb, NAME => 'TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T08:59:41,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1c52b94a9e6adce80d2d034c31b67cfb, NAME => 'TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1c52b94a9e6adce80d2d034c31b67cfb, NAME => 'TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T08:59:41,903 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/B/a727775cf6e1486e8724e7cedb5341a4 is 50, key is test_row_0/B:col10/1732093181400/Put/seqid=0 2024-11-20T08:59:41,912 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073741933_1109 (size=12301) 2024-11-20T08:59:41,913 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=479 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/B/a727775cf6e1486e8724e7cedb5341a4 2024-11-20T08:59:41,929 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/C/bb08f126678947bdb084af69a6eb82b1 is 50, key is test_row_0/C:col10/1732093181400/Put/seqid=0 2024-11-20T08:59:41,934 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073741934_1110 (size=12301) 2024-11-20T08:59:42,030 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:42,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43672 deadline: 1732093242027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:42,032 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:42,032 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:42,032 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:42,048 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-20T08:59:42,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb. 2024-11-20T08:59:42,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb. as already flushing 2024-11-20T08:59:42,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb. 2024-11-20T08:59:42,049 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 1c52b94a9e6adce80d2d034c31b67cfb, NAME => 'TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T08:59:42,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 1c52b94a9e6adce80d2d034c31b67cfb, NAME => 'TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T08:59:42,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1c52b94a9e6adce80d2d034c31b67cfb, NAME => 'TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1c52b94a9e6adce80d2d034c31b67cfb, NAME => 'TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T08:59:42,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-20T08:59:42,108 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:42,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43620 deadline: 1732093242107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:42,109 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:42,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43628 deadline: 1732093242108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:42,109 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:42,109 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:42,109 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:42,109 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:42,109 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:42,109 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:42,204 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-20T08:59:42,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb. 2024-11-20T08:59:42,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb. as already flushing 2024-11-20T08:59:42,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb. 2024-11-20T08:59:42,204 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 1c52b94a9e6adce80d2d034c31b67cfb, NAME => 'TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T08:59:42,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 1c52b94a9e6adce80d2d034c31b67cfb, NAME => 'TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T08:59:42,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1c52b94a9e6adce80d2d034c31b67cfb, NAME => 'TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1c52b94a9e6adce80d2d034c31b67cfb, NAME => 'TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T08:59:42,336 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=479 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/C/bb08f126678947bdb084af69a6eb82b1 2024-11-20T08:59:42,345 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/A/4f6e24e4ae594f2991ae0ecfcaba82d4 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/4f6e24e4ae594f2991ae0ecfcaba82d4 2024-11-20T08:59:42,353 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/4f6e24e4ae594f2991ae0ecfcaba82d4, entries=150, sequenceid=479, filesize=12.0 K 2024-11-20T08:59:42,355 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/B/a727775cf6e1486e8724e7cedb5341a4 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/a727775cf6e1486e8724e7cedb5341a4 2024-11-20T08:59:42,359 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-20T08:59:42,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb. 2024-11-20T08:59:42,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb. as already flushing 2024-11-20T08:59:42,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb. 2024-11-20T08:59:42,359 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 1c52b94a9e6adce80d2d034c31b67cfb, NAME => 'TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T08:59:42,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 1c52b94a9e6adce80d2d034c31b67cfb, NAME => 'TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T08:59:42,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1c52b94a9e6adce80d2d034c31b67cfb, NAME => 'TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1c52b94a9e6adce80d2d034c31b67cfb, NAME => 'TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T08:59:42,361 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/a727775cf6e1486e8724e7cedb5341a4, entries=150, sequenceid=479, filesize=12.0 K 2024-11-20T08:59:42,362 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/C/bb08f126678947bdb084af69a6eb82b1 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/C/bb08f126678947bdb084af69a6eb82b1 2024-11-20T08:59:42,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-20T08:59:42,370 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/C/bb08f126678947bdb084af69a6eb82b1, entries=150, sequenceid=479, filesize=12.0 K 2024-11-20T08:59:42,372 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~167.72 KB/171750, heapSize ~440.16 KB/450720, currentSize=40.25 KB/41220 for 1c52b94a9e6adce80d2d034c31b67cfb in 953ms, sequenceid=479, compaction requested=true 2024-11-20T08:59:42,372 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1c52b94a9e6adce80d2d034c31b67cfb: 2024-11-20T08:59:42,373 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1c52b94a9e6adce80d2d034c31b67cfb:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T08:59:42,373 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T08:59:42,373 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T08:59:42,373 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T08:59:42,375 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37925 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T08:59:42,375 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1541): 1c52b94a9e6adce80d2d034c31b67cfb/B is initiating minor compaction (all files) 2024-11-20T08:59:42,375 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1c52b94a9e6adce80d2d034c31b67cfb/B in TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb. 2024-11-20T08:59:42,375 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/3e5841ea82c54ab69aa11e2cd14fb1a9, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/51f3985067d2450eb0e0f202b6395189, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/a727775cf6e1486e8724e7cedb5341a4] into tmpdir=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp, totalSize=37.0 K 2024-11-20T08:59:42,375 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37925 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T08:59:42,375 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HStore(1541): 1c52b94a9e6adce80d2d034c31b67cfb/A is initiating minor compaction (all files) 2024-11-20T08:59:42,375 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting 3e5841ea82c54ab69aa11e2cd14fb1a9, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=438, earliestPutTs=1732093179448 2024-11-20T08:59:42,376 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1c52b94a9e6adce80d2d034c31b67cfb/A in TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb. 2024-11-20T08:59:42,376 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/3e40b48d7d93455abc46ce2974f19148, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/e0beb8f7442a40c48422b266e7d5df63, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/4f6e24e4ae594f2991ae0ecfcaba82d4] into tmpdir=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp, totalSize=37.0 K 2024-11-20T08:59:42,376 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting 51f3985067d2450eb0e0f202b6395189, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=448, earliestPutTs=1732093180631 2024-11-20T08:59:42,377 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting a727775cf6e1486e8724e7cedb5341a4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=479, earliestPutTs=1732093181392 2024-11-20T08:59:42,377 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3e40b48d7d93455abc46ce2974f19148, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=438, earliestPutTs=1732093179448 2024-11-20T08:59:42,377 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.Compactor(225): Compacting e0beb8f7442a40c48422b266e7d5df63, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=448, earliestPutTs=1732093180631 2024-11-20T08:59:42,379 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4f6e24e4ae594f2991ae0ecfcaba82d4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=479, earliestPutTs=1732093181392 2024-11-20T08:59:42,389 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1c52b94a9e6adce80d2d034c31b67cfb#B#compaction#98 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T08:59:42,390 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/B/a49c71f381b8482fa40d7e14457556a8 is 50, key is test_row_0/B:col10/1732093181400/Put/seqid=0 2024-11-20T08:59:42,374 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1c52b94a9e6adce80d2d034c31b67cfb:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T08:59:42,393 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T08:59:42,393 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1c52b94a9e6adce80d2d034c31b67cfb:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T08:59:42,393 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T08:59:42,410 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1c52b94a9e6adce80d2d034c31b67cfb#A#compaction#99 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T08:59:42,412 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/A/fe9200cfb452410cb92506c681345cba is 50, key is test_row_0/A:col10/1732093181400/Put/seqid=0 2024-11-20T08:59:42,466 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073741935_1111 (size=13425) 2024-11-20T08:59:42,474 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/B/a49c71f381b8482fa40d7e14457556a8 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/a49c71f381b8482fa40d7e14457556a8 2024-11-20T08:59:42,487 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1c52b94a9e6adce80d2d034c31b67cfb/B of 1c52b94a9e6adce80d2d034c31b67cfb into a49c71f381b8482fa40d7e14457556a8(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T08:59:42,488 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1c52b94a9e6adce80d2d034c31b67cfb: 2024-11-20T08:59:42,488 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., storeName=1c52b94a9e6adce80d2d034c31b67cfb/B, priority=13, startTime=1732093182373; duration=0sec 2024-11-20T08:59:42,489 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T08:59:42,489 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1c52b94a9e6adce80d2d034c31b67cfb:B 2024-11-20T08:59:42,489 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T08:59:42,489 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073741936_1112 (size=13425) 2024-11-20T08:59:42,498 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37891 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T08:59:42,498 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1541): 1c52b94a9e6adce80d2d034c31b67cfb/C is initiating minor compaction (all files) 2024-11-20T08:59:42,498 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1c52b94a9e6adce80d2d034c31b67cfb/C in TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb. 2024-11-20T08:59:42,499 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/A/fe9200cfb452410cb92506c681345cba as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/fe9200cfb452410cb92506c681345cba 2024-11-20T08:59:42,500 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/C/4b004e038ded48e585be6e88fd09a463, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/C/00aeea979ff341178675a0078c5616c2, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/C/bb08f126678947bdb084af69a6eb82b1] into tmpdir=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp, totalSize=37.0 K 2024-11-20T08:59:42,501 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting 4b004e038ded48e585be6e88fd09a463, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=438, earliestPutTs=1732093179448 2024-11-20T08:59:42,501 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting 00aeea979ff341178675a0078c5616c2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=448, earliestPutTs=1732093180631 2024-11-20T08:59:42,502 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting bb08f126678947bdb084af69a6eb82b1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=479, earliestPutTs=1732093181392 2024-11-20T08:59:42,511 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1c52b94a9e6adce80d2d034c31b67cfb/A of 1c52b94a9e6adce80d2d034c31b67cfb into fe9200cfb452410cb92506c681345cba(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T08:59:42,512 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1c52b94a9e6adce80d2d034c31b67cfb: 2024-11-20T08:59:42,512 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., storeName=1c52b94a9e6adce80d2d034c31b67cfb/A, priority=13, startTime=1732093182373; duration=0sec 2024-11-20T08:59:42,512 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T08:59:42,512 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1c52b94a9e6adce80d2d034c31b67cfb:A 2024-11-20T08:59:42,514 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-20T08:59:42,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb. 2024-11-20T08:59:42,514 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2902): Flushing 1c52b94a9e6adce80d2d034c31b67cfb 3/3 column families, dataSize=40.25 KB heapSize=106.22 KB 2024-11-20T08:59:42,526 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1c52b94a9e6adce80d2d034c31b67cfb#C#compaction#100 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T08:59:42,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/A/ae5926da1a5f4f67ba7cca9026fe41b7 is 50, key is test_row_0/A:col10/1732093181469/Put/seqid=0 2024-11-20T08:59:42,529 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/C/54d577d86aca418f91c9ac03730587ec is 50, key is test_row_0/C:col10/1732093181400/Put/seqid=0 2024-11-20T08:59:42,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(8855): Flush requested on 1c52b94a9e6adce80d2d034c31b67cfb 2024-11-20T08:59:42,545 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb. as already flushing 2024-11-20T08:59:42,548 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073741938_1114 (size=13391) 2024-11-20T08:59:42,558 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/C/54d577d86aca418f91c9ac03730587ec as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/C/54d577d86aca418f91c9ac03730587ec 2024-11-20T08:59:42,561 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073741937_1113 (size=9857) 2024-11-20T08:59:42,562 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=490 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/A/ae5926da1a5f4f67ba7cca9026fe41b7 2024-11-20T08:59:42,568 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1c52b94a9e6adce80d2d034c31b67cfb/C of 1c52b94a9e6adce80d2d034c31b67cfb into 54d577d86aca418f91c9ac03730587ec(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T08:59:42,568 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1c52b94a9e6adce80d2d034c31b67cfb: 2024-11-20T08:59:42,568 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., storeName=1c52b94a9e6adce80d2d034c31b67cfb/C, priority=13, startTime=1732093182393; duration=0sec 2024-11-20T08:59:42,568 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T08:59:42,568 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1c52b94a9e6adce80d2d034c31b67cfb:C 2024-11-20T08:59:42,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/B/04589ae207c7465698a5f1ef95c68f46 is 50, key is test_row_0/B:col10/1732093181469/Put/seqid=0 2024-11-20T08:59:42,597 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073741939_1115 (size=9857) 2024-11-20T08:59:42,657 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:42,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43628 deadline: 1732093242650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:42,658 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:42,658 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:42,658 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:42,658 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:42,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43672 deadline: 1732093242653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:42,659 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:42,659 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:42,659 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:42,660 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:42,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43620 deadline: 1732093242657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:42,661 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:42,661 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:42,661 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:42,770 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:42,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43620 deadline: 1732093242768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:42,770 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:42,771 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:42,771 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:42,771 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:42,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43672 deadline: 1732093242769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:42,772 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:42,772 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:42,772 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:42,772 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:42,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43628 deadline: 1732093242769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:42,773 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:42,773 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:42,773 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:42,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-20T08:59:42,980 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:42,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43620 deadline: 1732093242979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:42,981 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:42,981 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:42,981 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:42,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43672 deadline: 1732093242979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:42,981 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:42,982 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:42,982 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:42,982 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:42,983 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:42,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43628 deadline: 1732093242979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:42,984 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:42,984 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:42,984 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:42,998 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=490 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/B/04589ae207c7465698a5f1ef95c68f46 2024-11-20T08:59:43,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/C/f62fe1d10d6a4fbba8018dcb10f8e72d is 50, key is test_row_0/C:col10/1732093181469/Put/seqid=0 2024-11-20T08:59:43,037 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073741940_1116 (size=9857) 2024-11-20T08:59:43,288 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:43,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43620 deadline: 1732093243287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:43,289 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:43,289 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:43,289 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:43,289 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:43,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43672 deadline: 1732093243288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:43,290 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:43,290 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:43,290 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:43,291 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:43,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43628 deadline: 1732093243288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:43,292 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:43,292 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:43,292 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:43,439 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=490 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/C/f62fe1d10d6a4fbba8018dcb10f8e72d 2024-11-20T08:59:43,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/A/ae5926da1a5f4f67ba7cca9026fe41b7 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/ae5926da1a5f4f67ba7cca9026fe41b7 2024-11-20T08:59:43,454 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/ae5926da1a5f4f67ba7cca9026fe41b7, entries=100, sequenceid=490, filesize=9.6 K 2024-11-20T08:59:43,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/B/04589ae207c7465698a5f1ef95c68f46 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/04589ae207c7465698a5f1ef95c68f46 2024-11-20T08:59:43,462 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/04589ae207c7465698a5f1ef95c68f46, entries=100, sequenceid=490, filesize=9.6 K 2024-11-20T08:59:43,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/C/f62fe1d10d6a4fbba8018dcb10f8e72d as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/C/f62fe1d10d6a4fbba8018dcb10f8e72d 2024-11-20T08:59:43,474 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/C/f62fe1d10d6a4fbba8018dcb10f8e72d, entries=100, sequenceid=490, filesize=9.6 K 2024-11-20T08:59:43,475 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~106.17 KB/108720, currentSize=161.02 KB/164880 for 1c52b94a9e6adce80d2d034c31b67cfb in 961ms, sequenceid=490, compaction requested=false 2024-11-20T08:59:43,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2603): Flush status journal for 1c52b94a9e6adce80d2d034c31b67cfb: 2024-11-20T08:59:43,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb. 2024-11-20T08:59:43,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=20 2024-11-20T08:59:43,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.HMaster(4169): Remote procedure done, pid=20 2024-11-20T08:59:43,487 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-11-20T08:59:43,487 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7400 sec 2024-11-20T08:59:43,490 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees in 1.7510 sec 2024-11-20T08:59:43,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(8855): Flush requested on 1c52b94a9e6adce80d2d034c31b67cfb 2024-11-20T08:59:43,801 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T08:59:43,801 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1c52b94a9e6adce80d2d034c31b67cfb 3/3 column families, dataSize=167.72 KB heapSize=440.20 KB 2024-11-20T08:59:43,810 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:43,810 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/A/531c4eb5eb21473287f979f0937cc4fd is 50, key is test_row_0/A:col10/1732093182655/Put/seqid=0 2024-11-20T08:59:43,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43620 deadline: 1732093243805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:43,811 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:43,811 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:43,811 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:43,813 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:43,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43672 deadline: 1732093243811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:43,814 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:43,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43628 deadline: 1732093243811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:43,814 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:43,814 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:43,814 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:43,819 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:43,819 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:43,819 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:43,829 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073741941_1117 (size=17181) 2024-11-20T08:59:43,830 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=60.38 KB at sequenceid=521 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/A/531c4eb5eb21473287f979f0937cc4fd 2024-11-20T08:59:43,846 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/B/be2c0a116116477a9e070a1ccbf4be2c is 50, key is test_row_0/B:col10/1732093182655/Put/seqid=0 2024-11-20T08:59:43,862 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073741942_1118 (size=12301) 2024-11-20T08:59:43,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-20T08:59:43,887 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T08:59:43,888 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T08:59:43,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] procedure2.ProcedureExecutor(1139): Stored pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees 2024-11-20T08:59:43,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-20T08:59:43,891 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T08:59:43,892 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T08:59:43,892 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T08:59:43,917 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:43,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43620 deadline: 1732093243917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:43,918 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:43,918 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:43,918 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:43,918 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:43,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43672 deadline: 1732093243917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:43,919 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:43,919 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:43,919 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:43,929 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:43,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43628 deadline: 1732093243927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:43,929 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:43,929 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:43,929 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:43,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-20T08:59:44,047 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-20T08:59:44,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb. 2024-11-20T08:59:44,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb. as already flushing 2024-11-20T08:59:44,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb. 2024-11-20T08:59:44,048 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 1c52b94a9e6adce80d2d034c31b67cfb, NAME => 'TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T08:59:44,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 1c52b94a9e6adce80d2d034c31b67cfb, NAME => 'TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T08:59:44,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1c52b94a9e6adce80d2d034c31b67cfb, NAME => 'TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1c52b94a9e6adce80d2d034c31b67cfb, NAME => 'TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T08:59:44,129 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:44,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43620 deadline: 1732093244127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:44,130 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:44,130 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:44,131 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:44,131 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:44,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43672 deadline: 1732093244127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:44,131 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:44,131 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:44,131 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:44,142 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:44,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43628 deadline: 1732093244137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:44,145 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:44,145 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:44,145 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:44,203 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-20T08:59:44,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb. 2024-11-20T08:59:44,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb. as already flushing 2024-11-20T08:59:44,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb. 2024-11-20T08:59:44,203 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 1c52b94a9e6adce80d2d034c31b67cfb, NAME => 'TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T08:59:44,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 1c52b94a9e6adce80d2d034c31b67cfb, NAME => 'TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T08:59:44,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1c52b94a9e6adce80d2d034c31b67cfb, NAME => 'TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1c52b94a9e6adce80d2d034c31b67cfb, NAME => 'TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T08:59:44,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-20T08:59:44,263 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=60.38 KB at sequenceid=521 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/B/be2c0a116116477a9e070a1ccbf4be2c 2024-11-20T08:59:44,282 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/C/d4bb1b4b77a848b6b4d2c9cf45aa6601 is 50, key is test_row_0/C:col10/1732093182655/Put/seqid=0 2024-11-20T08:59:44,315 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073741943_1119 (size=12301) 2024-11-20T08:59:44,319 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=60.38 KB at sequenceid=521 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/C/d4bb1b4b77a848b6b4d2c9cf45aa6601 2024-11-20T08:59:44,331 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/A/531c4eb5eb21473287f979f0937cc4fd as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/531c4eb5eb21473287f979f0937cc4fd 2024-11-20T08:59:44,354 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/531c4eb5eb21473287f979f0937cc4fd, entries=250, sequenceid=521, filesize=16.8 K 2024-11-20T08:59:44,356 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/B/be2c0a116116477a9e070a1ccbf4be2c as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/be2c0a116116477a9e070a1ccbf4be2c 2024-11-20T08:59:44,357 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-20T08:59:44,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb. 2024-11-20T08:59:44,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb. as already flushing 2024-11-20T08:59:44,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb. 2024-11-20T08:59:44,358 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 1c52b94a9e6adce80d2d034c31b67cfb, NAME => 'TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T08:59:44,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 1c52b94a9e6adce80d2d034c31b67cfb, NAME => 'TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T08:59:44,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1c52b94a9e6adce80d2d034c31b67cfb, NAME => 'TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1c52b94a9e6adce80d2d034c31b67cfb, NAME => 'TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T08:59:44,372 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/be2c0a116116477a9e070a1ccbf4be2c, entries=150, sequenceid=521, filesize=12.0 K 2024-11-20T08:59:44,374 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/C/d4bb1b4b77a848b6b4d2c9cf45aa6601 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/C/d4bb1b4b77a848b6b4d2c9cf45aa6601 2024-11-20T08:59:44,387 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/C/d4bb1b4b77a848b6b4d2c9cf45aa6601, entries=150, sequenceid=521, filesize=12.0 K 2024-11-20T08:59:44,389 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~181.14 KB/185490, heapSize ~475.31 KB/486720, currentSize=20.13 KB/20610 for 1c52b94a9e6adce80d2d034c31b67cfb in 588ms, sequenceid=521, compaction requested=true 2024-11-20T08:59:44,389 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1c52b94a9e6adce80d2d034c31b67cfb: 2024-11-20T08:59:44,389 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1c52b94a9e6adce80d2d034c31b67cfb:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T08:59:44,389 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T08:59:44,389 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1c52b94a9e6adce80d2d034c31b67cfb:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T08:59:44,389 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T08:59:44,389 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T08:59:44,390 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1c52b94a9e6adce80d2d034c31b67cfb:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T08:59:44,390 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-20T08:59:44,390 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T08:59:44,396 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40463 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T08:59:44,397 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HStore(1541): 1c52b94a9e6adce80d2d034c31b67cfb/A is initiating minor compaction (all files) 2024-11-20T08:59:44,397 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1c52b94a9e6adce80d2d034c31b67cfb/A in TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb. 2024-11-20T08:59:44,397 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/fe9200cfb452410cb92506c681345cba, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/ae5926da1a5f4f67ba7cca9026fe41b7, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/531c4eb5eb21473287f979f0937cc4fd] into tmpdir=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp, totalSize=39.5 K 2024-11-20T08:59:44,397 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35583 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T08:59:44,397 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1541): 1c52b94a9e6adce80d2d034c31b67cfb/B is initiating minor compaction (all files) 2024-11-20T08:59:44,397 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1c52b94a9e6adce80d2d034c31b67cfb/B in TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb. 2024-11-20T08:59:44,398 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/a49c71f381b8482fa40d7e14457556a8, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/04589ae207c7465698a5f1ef95c68f46, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/be2c0a116116477a9e070a1ccbf4be2c] into tmpdir=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp, totalSize=34.7 K 2024-11-20T08:59:44,398 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.Compactor(225): Compacting fe9200cfb452410cb92506c681345cba, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=479, earliestPutTs=1732093181392 2024-11-20T08:59:44,398 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting a49c71f381b8482fa40d7e14457556a8, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=479, earliestPutTs=1732093181392 2024-11-20T08:59:44,398 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.Compactor(225): Compacting ae5926da1a5f4f67ba7cca9026fe41b7, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=490, earliestPutTs=1732093181469 2024-11-20T08:59:44,399 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting 04589ae207c7465698a5f1ef95c68f46, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=490, earliestPutTs=1732093181469 2024-11-20T08:59:44,399 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.Compactor(225): Compacting 531c4eb5eb21473287f979f0937cc4fd, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=521, earliestPutTs=1732093182651 2024-11-20T08:59:44,399 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting be2c0a116116477a9e070a1ccbf4be2c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=521, earliestPutTs=1732093182655 2024-11-20T08:59:44,418 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1c52b94a9e6adce80d2d034c31b67cfb#B#compaction#107 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T08:59:44,419 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/B/c9b832fc7a774b949512f1419aa0864a is 50, key is test_row_0/B:col10/1732093182655/Put/seqid=0 2024-11-20T08:59:44,426 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1c52b94a9e6adce80d2d034c31b67cfb#A#compaction#108 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T08:59:44,427 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/A/a08e9bdcb5a94be4b339121b105e1a6f is 50, key is test_row_0/A:col10/1732093182655/Put/seqid=0 2024-11-20T08:59:44,443 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073741944_1120 (size=13527) 2024-11-20T08:59:44,458 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/B/c9b832fc7a774b949512f1419aa0864a as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/c9b832fc7a774b949512f1419aa0864a 2024-11-20T08:59:44,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(8855): Flush requested on 1c52b94a9e6adce80d2d034c31b67cfb 2024-11-20T08:59:44,459 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T08:59:44,459 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1c52b94a9e6adce80d2d034c31b67cfb 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-20T08:59:44,465 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1c52b94a9e6adce80d2d034c31b67cfb/B of 1c52b94a9e6adce80d2d034c31b67cfb into c9b832fc7a774b949512f1419aa0864a(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T08:59:44,465 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1c52b94a9e6adce80d2d034c31b67cfb: 2024-11-20T08:59:44,466 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., storeName=1c52b94a9e6adce80d2d034c31b67cfb/B, priority=13, startTime=1732093184389; duration=0sec 2024-11-20T08:59:44,466 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T08:59:44,466 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1c52b94a9e6adce80d2d034c31b67cfb:B 2024-11-20T08:59:44,466 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T08:59:44,468 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35549 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T08:59:44,468 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1541): 1c52b94a9e6adce80d2d034c31b67cfb/C is initiating minor compaction (all files) 2024-11-20T08:59:44,468 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1c52b94a9e6adce80d2d034c31b67cfb/C in TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb. 2024-11-20T08:59:44,468 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/C/54d577d86aca418f91c9ac03730587ec, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/C/f62fe1d10d6a4fbba8018dcb10f8e72d, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/C/d4bb1b4b77a848b6b4d2c9cf45aa6601] into tmpdir=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp, totalSize=34.7 K 2024-11-20T08:59:44,469 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting 54d577d86aca418f91c9ac03730587ec, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=479, earliestPutTs=1732093181392 2024-11-20T08:59:44,470 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting f62fe1d10d6a4fbba8018dcb10f8e72d, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=490, earliestPutTs=1732093181469 2024-11-20T08:59:44,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073741945_1121 (size=13527) 2024-11-20T08:59:44,473 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting d4bb1b4b77a848b6b4d2c9cf45aa6601, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=521, earliestPutTs=1732093182655 2024-11-20T08:59:44,479 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/A/a08e9bdcb5a94be4b339121b105e1a6f as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/a08e9bdcb5a94be4b339121b105e1a6f 2024-11-20T08:59:44,484 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/A/dfbb48b4b3e140319367aa954143dbd8 is 50, key is test_row_0/A:col10/1732093184447/Put/seqid=0 2024-11-20T08:59:44,487 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1c52b94a9e6adce80d2d034c31b67cfb/A of 1c52b94a9e6adce80d2d034c31b67cfb into a08e9bdcb5a94be4b339121b105e1a6f(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T08:59:44,487 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1c52b94a9e6adce80d2d034c31b67cfb: 2024-11-20T08:59:44,487 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., storeName=1c52b94a9e6adce80d2d034c31b67cfb/A, priority=13, startTime=1732093184389; duration=0sec 2024-11-20T08:59:44,487 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T08:59:44,487 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1c52b94a9e6adce80d2d034c31b67cfb:A 2024-11-20T08:59:44,494 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1c52b94a9e6adce80d2d034c31b67cfb#C#compaction#110 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T08:59:44,495 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/C/9966d23756d94aaf9e69dd2c5b2c911e is 50, key is test_row_0/C:col10/1732093182655/Put/seqid=0 2024-11-20T08:59:44,512 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-20T08:59:44,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb. 2024-11-20T08:59:44,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb. as already flushing 2024-11-20T08:59:44,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb. 2024-11-20T08:59:44,512 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 1c52b94a9e6adce80d2d034c31b67cfb, NAME => 'TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T08:59:44,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 1c52b94a9e6adce80d2d034c31b67cfb, NAME => 'TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T08:59:44,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1c52b94a9e6adce80d2d034c31b67cfb, NAME => 'TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1c52b94a9e6adce80d2d034c31b67cfb, NAME => 'TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T08:59:44,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-20T08:59:44,547 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073741946_1122 (size=14741) 2024-11-20T08:59:44,550 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=532 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/A/dfbb48b4b3e140319367aa954143dbd8 2024-11-20T08:59:44,566 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:44,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43620 deadline: 1732093244559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:44,567 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:44,567 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:44,567 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:44,567 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:44,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43672 deadline: 1732093244565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:44,568 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:44,568 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:44,568 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:44,587 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073741947_1123 (size=13493) 2024-11-20T08:59:44,576 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:44,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43628 deadline: 1732093244567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:44,589 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:44,589 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:44,589 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:44,595 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/B/987c1b45588d47ddbf81c183342babbb is 50, key is test_row_0/B:col10/1732093184447/Put/seqid=0 2024-11-20T08:59:44,665 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073741948_1124 (size=12301) 2024-11-20T08:59:44,667 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=532 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/B/987c1b45588d47ddbf81c183342babbb 2024-11-20T08:59:44,668 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-20T08:59:44,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb. 2024-11-20T08:59:44,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb. as already flushing 2024-11-20T08:59:44,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb. 2024-11-20T08:59:44,669 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 1c52b94a9e6adce80d2d034c31b67cfb, NAME => 'TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T08:59:44,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 1c52b94a9e6adce80d2d034c31b67cfb, NAME => 'TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T08:59:44,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1c52b94a9e6adce80d2d034c31b67cfb, NAME => 'TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1c52b94a9e6adce80d2d034c31b67cfb, NAME => 'TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T08:59:44,684 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:44,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43672 deadline: 1732093244678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:44,685 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:44,685 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:44,685 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:44,685 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:44,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43620 deadline: 1732093244678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:44,686 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:44,686 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:44,686 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:44,694 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/C/1052f6328e014379aeaf32a1555716fb is 50, key is test_row_0/C:col10/1732093184447/Put/seqid=0 2024-11-20T08:59:44,698 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:44,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43628 deadline: 1732093244697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:44,699 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:44,699 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:44,699 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:44,725 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073741949_1125 (size=12301) 2024-11-20T08:59:44,726 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=532 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/C/1052f6328e014379aeaf32a1555716fb 2024-11-20T08:59:44,736 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/A/dfbb48b4b3e140319367aa954143dbd8 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/dfbb48b4b3e140319367aa954143dbd8 2024-11-20T08:59:44,741 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/dfbb48b4b3e140319367aa954143dbd8, entries=200, sequenceid=532, filesize=14.4 K 2024-11-20T08:59:44,742 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/B/987c1b45588d47ddbf81c183342babbb as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/987c1b45588d47ddbf81c183342babbb 2024-11-20T08:59:44,758 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/987c1b45588d47ddbf81c183342babbb, entries=150, sequenceid=532, filesize=12.0 K 2024-11-20T08:59:44,759 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/C/1052f6328e014379aeaf32a1555716fb as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/C/1052f6328e014379aeaf32a1555716fb 2024-11-20T08:59:44,769 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/C/1052f6328e014379aeaf32a1555716fb, entries=150, sequenceid=532, filesize=12.0 K 2024-11-20T08:59:44,770 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 1c52b94a9e6adce80d2d034c31b67cfb in 310ms, sequenceid=532, compaction requested=false 2024-11-20T08:59:44,770 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1c52b94a9e6adce80d2d034c31b67cfb: 2024-11-20T08:59:44,822 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-20T08:59:44,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb. 2024-11-20T08:59:44,823 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2902): Flushing 1c52b94a9e6adce80d2d034c31b67cfb 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-20T08:59:44,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/A/7f4b8131bbd641fbaad7a4a91ab244db is 50, key is test_row_0/A:col10/1732093184564/Put/seqid=0 2024-11-20T08:59:44,861 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073741950_1126 (size=12301) 2024-11-20T08:59:44,862 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=559 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/A/7f4b8131bbd641fbaad7a4a91ab244db 2024-11-20T08:59:44,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/B/e2447008ae3b4e37bd4e0da3466845f9 is 50, key is test_row_0/B:col10/1732093184564/Put/seqid=0 2024-11-20T08:59:44,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(8855): Flush requested on 1c52b94a9e6adce80d2d034c31b67cfb 2024-11-20T08:59:44,900 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb. as already flushing 2024-11-20T08:59:44,938 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:44,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43620 deadline: 1732093244929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:44,938 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:44,939 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:44,939 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:44,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43672 deadline: 1732093244929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:44,939 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:44,939 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:44,939 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:44,939 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:44,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43628 deadline: 1732093244930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:44,939 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:44,940 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:44,940 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:44,940 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:44,942 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073741951_1127 (size=12301) 2024-11-20T08:59:45,001 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/C/9966d23756d94aaf9e69dd2c5b2c911e as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/C/9966d23756d94aaf9e69dd2c5b2c911e 2024-11-20T08:59:45,008 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1c52b94a9e6adce80d2d034c31b67cfb/C of 1c52b94a9e6adce80d2d034c31b67cfb into 9966d23756d94aaf9e69dd2c5b2c911e(size=13.2 K), total size for store is 25.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T08:59:45,008 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1c52b94a9e6adce80d2d034c31b67cfb: 2024-11-20T08:59:45,009 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., storeName=1c52b94a9e6adce80d2d034c31b67cfb/C, priority=13, startTime=1732093184389; duration=0sec 2024-11-20T08:59:45,009 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T08:59:45,009 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1c52b94a9e6adce80d2d034c31b67cfb:C 2024-11-20T08:59:45,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-20T08:59:45,049 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:45,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43672 deadline: 1732093245047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:45,050 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:45,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43620 deadline: 1732093245047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:45,050 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:45,050 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:45,050 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:45,051 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:45,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43628 deadline: 1732093245047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:45,051 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:45,051 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:45,051 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:45,051 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:45,051 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:45,051 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:45,258 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:45,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43672 deadline: 1732093245257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:45,259 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:45,259 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:45,259 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:45,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43628 deadline: 1732093245257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:45,259 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:45,259 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:45,259 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:45,260 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:45,260 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:45,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43620 deadline: 1732093245257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:45,260 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:45,260 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:45,260 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:45,343 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=559 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/B/e2447008ae3b4e37bd4e0da3466845f9 2024-11-20T08:59:45,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/C/ec28ad95e4954444a65fec1c589debfd is 50, key is test_row_0/C:col10/1732093184564/Put/seqid=0 2024-11-20T08:59:45,360 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073741952_1128 (size=12301) 2024-11-20T08:59:45,362 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=559 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/C/ec28ad95e4954444a65fec1c589debfd 2024-11-20T08:59:45,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/A/7f4b8131bbd641fbaad7a4a91ab244db as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/7f4b8131bbd641fbaad7a4a91ab244db 2024-11-20T08:59:45,377 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/7f4b8131bbd641fbaad7a4a91ab244db, entries=150, sequenceid=559, filesize=12.0 K 2024-11-20T08:59:45,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/B/e2447008ae3b4e37bd4e0da3466845f9 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/e2447008ae3b4e37bd4e0da3466845f9 2024-11-20T08:59:45,396 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/e2447008ae3b4e37bd4e0da3466845f9, entries=150, sequenceid=559, filesize=12.0 K 2024-11-20T08:59:45,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/C/ec28ad95e4954444a65fec1c589debfd as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/C/ec28ad95e4954444a65fec1c589debfd 2024-11-20T08:59:45,409 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/C/ec28ad95e4954444a65fec1c589debfd, entries=150, sequenceid=559, filesize=12.0 K 2024-11-20T08:59:45,410 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 1c52b94a9e6adce80d2d034c31b67cfb in 588ms, sequenceid=559, compaction requested=true 2024-11-20T08:59:45,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2603): Flush status journal for 1c52b94a9e6adce80d2d034c31b67cfb: 2024-11-20T08:59:45,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb. 2024-11-20T08:59:45,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=22 2024-11-20T08:59:45,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.HMaster(4169): Remote procedure done, pid=22 2024-11-20T08:59:45,414 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-11-20T08:59:45,414 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5200 sec 2024-11-20T08:59:45,419 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees in 1.5280 sec 2024-11-20T08:59:45,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(8855): Flush requested on 1c52b94a9e6adce80d2d034c31b67cfb 2024-11-20T08:59:45,569 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T08:59:45,569 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1c52b94a9e6adce80d2d034c31b67cfb 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-20T08:59:45,576 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/A/55d3a4bb481249be96ff177bf1b85042 is 50, key is test_row_0/A:col10/1732093184925/Put/seqid=0 2024-11-20T08:59:45,602 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073741953_1129 (size=14741) 2024-11-20T08:59:45,618 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:45,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43628 deadline: 1732093245613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:45,619 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:45,619 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:45,619 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:45,622 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:45,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43672 deadline: 1732093245614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:45,623 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:45,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43620 deadline: 1732093245615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:45,623 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:45,624 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:45,624 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:45,624 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:45,624 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:45,624 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:45,729 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:45,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43628 deadline: 1732093245727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:45,730 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:45,730 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:45,730 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:45,730 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:45,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43620 deadline: 1732093245727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:45,730 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:45,730 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:45,730 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:45,730 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:45,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43672 deadline: 1732093245727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:45,731 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:45,731 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:45,731 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:45,831 INFO [Thread-156 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-156. 2024-11-20T08:59:45,831 DEBUG [Thread-156 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T08:59:45,833 INFO [Thread-158 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-158. 2024-11-20T08:59:45,834 DEBUG [Thread-158 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T08:59:45,834 INFO [Thread-154 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-154. 2024-11-20T08:59:45,835 DEBUG [Thread-154 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T08:59:45,836 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T08:59:45,836 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T08:59:45,837 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T08:59:45,837 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T08:59:45,837 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T08:59:45,837 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T08:59:45,837 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T08:59:45,837 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T08:59:45,838 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T08:59:45,839 INFO [Thread-160 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-160. 2024-11-20T08:59:45,839 DEBUG [Thread-160 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T08:59:45,839 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T08:59:45,839 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T08:59:45,839 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T08:59:45,907 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:45,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43644 deadline: 1732093245907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:45,908 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to be1eb2620e0e,34223,1732093159960 for 'test_row_2' in 1c52b94a9e6adce80d2d034c31b67cfb of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8217 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-20T08:59:45,909 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:45,909 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:45,909 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:45,938 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:45,938 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:45,938 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:45,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43620 deadline: 1732093245937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:45,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43608 deadline: 1732093245937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:45,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43628 deadline: 1732093245937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:45,938 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:45,938 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:45,938 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:45,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43672 deadline: 1732093245938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:45,938 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:45,938 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:45,939 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:45,939 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:45,939 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:45,939 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:45,939 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:45,938 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to be1eb2620e0e,34223,1732093159960 for 'test_row_2' in 1c52b94a9e6adce80d2d034c31b67cfb of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8247 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-20T08:59:45,939 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:45,939 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:45,939 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:46,003 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=572 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/A/55d3a4bb481249be96ff177bf1b85042 2024-11-20T08:59:46,011 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/B/f15f5983f4544ea5a9a2aca942ecc63e is 50, key is test_row_0/B:col10/1732093184925/Put/seqid=0 2024-11-20T08:59:46,015 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073741954_1130 (size=12301) 2024-11-20T08:59:46,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-20T08:59:46,038 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T08:59:46,247 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:46,247 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:46,247 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:46,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43620 deadline: 1732093246247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:46,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43628 deadline: 1732093246247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:46,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43672 deadline: 1732093246247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:46,248 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:46,248 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:46,248 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:46,249 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:46,249 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:46,249 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:46,249 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:46,249 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:46,249 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:46,416 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=572 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/B/f15f5983f4544ea5a9a2aca942ecc63e 2024-11-20T08:59:46,424 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/C/471366db1eb7492f9363719639f52cd9 is 50, key is test_row_0/C:col10/1732093184925/Put/seqid=0 2024-11-20T08:59:46,428 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073741955_1131 (size=12301) 2024-11-20T08:59:46,758 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:46,758 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:46,758 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T08:59:46,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43620 deadline: 1732093246758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:46,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43628 deadline: 1732093246757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:46,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43672 deadline: 1732093246757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:46,759 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:46,759 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:46,759 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:46,759 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:46,759 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:46,759 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:46,759 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T08:59:46,759 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:46,759 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T08:59:46,829 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=572 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/C/471366db1eb7492f9363719639f52cd9 2024-11-20T08:59:46,835 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/A/55d3a4bb481249be96ff177bf1b85042 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/55d3a4bb481249be96ff177bf1b85042 2024-11-20T08:59:46,840 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/55d3a4bb481249be96ff177bf1b85042, entries=200, sequenceid=572, filesize=14.4 K 2024-11-20T08:59:46,841 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/B/f15f5983f4544ea5a9a2aca942ecc63e as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/f15f5983f4544ea5a9a2aca942ecc63e 2024-11-20T08:59:46,847 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/f15f5983f4544ea5a9a2aca942ecc63e, entries=150, sequenceid=572, filesize=12.0 K 2024-11-20T08:59:46,848 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/C/471366db1eb7492f9363719639f52cd9 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/C/471366db1eb7492f9363719639f52cd9 2024-11-20T08:59:46,853 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/C/471366db1eb7492f9363719639f52cd9, entries=150, sequenceid=572, filesize=12.0 K 2024-11-20T08:59:46,854 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 1c52b94a9e6adce80d2d034c31b67cfb in 1285ms, sequenceid=572, compaction requested=true 2024-11-20T08:59:46,855 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1c52b94a9e6adce80d2d034c31b67cfb: 2024-11-20T08:59:46,855 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1c52b94a9e6adce80d2d034c31b67cfb:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T08:59:46,855 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T08:59:46,855 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T08:59:46,855 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1c52b94a9e6adce80d2d034c31b67cfb:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T08:59:46,855 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T08:59:46,855 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1c52b94a9e6adce80d2d034c31b67cfb:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T08:59:46,855 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T08:59:46,855 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T08:59:46,856 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 55310 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T08:59:46,857 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50430 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T08:59:46,857 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HStore(1541): 1c52b94a9e6adce80d2d034c31b67cfb/A is initiating minor compaction (all files) 2024-11-20T08:59:46,857 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1541): 1c52b94a9e6adce80d2d034c31b67cfb/B is initiating minor compaction (all files) 2024-11-20T08:59:46,857 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1c52b94a9e6adce80d2d034c31b67cfb/A in TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb. 2024-11-20T08:59:46,857 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1c52b94a9e6adce80d2d034c31b67cfb/B in TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb. 2024-11-20T08:59:46,857 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/a08e9bdcb5a94be4b339121b105e1a6f, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/dfbb48b4b3e140319367aa954143dbd8, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/7f4b8131bbd641fbaad7a4a91ab244db, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/55d3a4bb481249be96ff177bf1b85042] into tmpdir=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp, totalSize=54.0 K 2024-11-20T08:59:46,857 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/c9b832fc7a774b949512f1419aa0864a, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/987c1b45588d47ddbf81c183342babbb, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/e2447008ae3b4e37bd4e0da3466845f9, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/f15f5983f4544ea5a9a2aca942ecc63e] into tmpdir=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp, totalSize=49.2 K 2024-11-20T08:59:46,857 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting c9b832fc7a774b949512f1419aa0864a, keycount=150, bloomtype=ROW, size=13.2 K, encoding=NONE, compression=NONE, seqNum=521, earliestPutTs=1732093182655 2024-11-20T08:59:46,857 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.Compactor(225): Compacting a08e9bdcb5a94be4b339121b105e1a6f, keycount=150, bloomtype=ROW, size=13.2 K, encoding=NONE, compression=NONE, seqNum=521, earliestPutTs=1732093182655 2024-11-20T08:59:46,858 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.Compactor(225): Compacting dfbb48b4b3e140319367aa954143dbd8, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=532, earliestPutTs=1732093183808 2024-11-20T08:59:46,858 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting 987c1b45588d47ddbf81c183342babbb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=532, earliestPutTs=1732093183808 2024-11-20T08:59:46,858 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting e2447008ae3b4e37bd4e0da3466845f9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=559, earliestPutTs=1732093184549 2024-11-20T08:59:46,858 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7f4b8131bbd641fbaad7a4a91ab244db, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=559, earliestPutTs=1732093184549 2024-11-20T08:59:46,859 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.Compactor(225): Compacting 55d3a4bb481249be96ff177bf1b85042, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=572, earliestPutTs=1732093184901 2024-11-20T08:59:46,859 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting f15f5983f4544ea5a9a2aca942ecc63e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=572, earliestPutTs=1732093184901 2024-11-20T08:59:46,869 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1c52b94a9e6adce80d2d034c31b67cfb#B#compaction#119 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T08:59:46,869 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/B/8018f60f25be47dbbeb0e69ea98d36b7 is 50, key is test_row_0/B:col10/1732093184925/Put/seqid=0 2024-11-20T08:59:46,871 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1c52b94a9e6adce80d2d034c31b67cfb#A#compaction#120 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T08:59:46,872 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/A/243f2dc7d8b6433bb1658703919b6b00 is 50, key is test_row_0/A:col10/1732093184925/Put/seqid=0 2024-11-20T08:59:46,877 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073741956_1132 (size=13663) 2024-11-20T08:59:46,879 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073741957_1133 (size=13663) 2024-11-20T08:59:46,885 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/A/243f2dc7d8b6433bb1658703919b6b00 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/243f2dc7d8b6433bb1658703919b6b00 2024-11-20T08:59:46,890 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1c52b94a9e6adce80d2d034c31b67cfb/A of 1c52b94a9e6adce80d2d034c31b67cfb into 243f2dc7d8b6433bb1658703919b6b00(size=13.3 K), total size for store is 13.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T08:59:46,890 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1c52b94a9e6adce80d2d034c31b67cfb: 2024-11-20T08:59:46,890 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., storeName=1c52b94a9e6adce80d2d034c31b67cfb/A, priority=12, startTime=1732093186855; duration=0sec 2024-11-20T08:59:46,890 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T08:59:46,890 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1c52b94a9e6adce80d2d034c31b67cfb:A 2024-11-20T08:59:46,891 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T08:59:46,892 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50396 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T08:59:46,892 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HStore(1541): 1c52b94a9e6adce80d2d034c31b67cfb/C is initiating minor compaction (all files) 2024-11-20T08:59:46,892 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1c52b94a9e6adce80d2d034c31b67cfb/C in TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb. 2024-11-20T08:59:46,892 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/C/9966d23756d94aaf9e69dd2c5b2c911e, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/C/1052f6328e014379aeaf32a1555716fb, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/C/ec28ad95e4954444a65fec1c589debfd, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/C/471366db1eb7492f9363719639f52cd9] into tmpdir=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp, totalSize=49.2 K 2024-11-20T08:59:46,892 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9966d23756d94aaf9e69dd2c5b2c911e, keycount=150, bloomtype=ROW, size=13.2 K, encoding=NONE, compression=NONE, seqNum=521, earliestPutTs=1732093182655 2024-11-20T08:59:46,893 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1052f6328e014379aeaf32a1555716fb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=532, earliestPutTs=1732093183808 2024-11-20T08:59:46,893 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.Compactor(225): Compacting ec28ad95e4954444a65fec1c589debfd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=559, earliestPutTs=1732093184549 2024-11-20T08:59:46,893 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.Compactor(225): Compacting 471366db1eb7492f9363719639f52cd9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=572, earliestPutTs=1732093184901 2024-11-20T08:59:46,902 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1c52b94a9e6adce80d2d034c31b67cfb#C#compaction#121 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T08:59:46,902 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/C/e0cc30253b754f00a46aa1ae26b9443a is 50, key is test_row_0/C:col10/1732093184925/Put/seqid=0 2024-11-20T08:59:46,906 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073741958_1134 (size=13629) 2024-11-20T08:59:47,284 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/B/8018f60f25be47dbbeb0e69ea98d36b7 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/8018f60f25be47dbbeb0e69ea98d36b7 2024-11-20T08:59:47,289 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1c52b94a9e6adce80d2d034c31b67cfb/B of 1c52b94a9e6adce80d2d034c31b67cfb into 8018f60f25be47dbbeb0e69ea98d36b7(size=13.3 K), total size for store is 13.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T08:59:47,289 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1c52b94a9e6adce80d2d034c31b67cfb: 2024-11-20T08:59:47,290 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., storeName=1c52b94a9e6adce80d2d034c31b67cfb/B, priority=12, startTime=1732093186855; duration=0sec 2024-11-20T08:59:47,290 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T08:59:47,290 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1c52b94a9e6adce80d2d034c31b67cfb:B 2024-11-20T08:59:47,312 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/C/e0cc30253b754f00a46aa1ae26b9443a as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/C/e0cc30253b754f00a46aa1ae26b9443a 2024-11-20T08:59:47,317 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1c52b94a9e6adce80d2d034c31b67cfb/C of 1c52b94a9e6adce80d2d034c31b67cfb into e0cc30253b754f00a46aa1ae26b9443a(size=13.3 K), total size for store is 13.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T08:59:47,317 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1c52b94a9e6adce80d2d034c31b67cfb: 2024-11-20T08:59:47,317 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., storeName=1c52b94a9e6adce80d2d034c31b67cfb/C, priority=12, startTime=1732093186855; duration=0sec 2024-11-20T08:59:47,317 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T08:59:47,318 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1c52b94a9e6adce80d2d034c31b67cfb:C 2024-11-20T08:59:47,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(8855): Flush requested on 1c52b94a9e6adce80d2d034c31b67cfb 2024-11-20T08:59:47,769 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T08:59:47,769 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1c52b94a9e6adce80d2d034c31b67cfb 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-20T08:59:47,769 INFO [Thread-147 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-147. 2024-11-20T08:59:47,770 INFO [Thread-145 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-145. 2024-11-20T08:59:47,770 DEBUG [Thread-147 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T08:59:47,770 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T08:59:47,770 DEBUG [Thread-145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T08:59:47,770 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T08:59:47,770 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T08:59:47,770 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T08:59:47,770 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T08:59:47,770 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T08:59:47,774 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/A/60c87dc06d614acda9e8c53e10631487 is 50, key is test_row_0/A:col10/1732093185611/Put/seqid=0 2024-11-20T08:59:47,778 INFO [Thread-151 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-151. 2024-11-20T08:59:47,778 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073741959_1135 (size=12301) 2024-11-20T08:59:47,778 DEBUG [Thread-151 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T08:59:47,778 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T08:59:47,778 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T08:59:47,778 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T08:59:48,179 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=601 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/A/60c87dc06d614acda9e8c53e10631487 2024-11-20T08:59:48,187 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/B/0e11491d25ef497aaeccfdffbd9ada72 is 50, key is test_row_0/B:col10/1732093185611/Put/seqid=0 2024-11-20T08:59:48,191 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073741960_1136 (size=12301) 2024-11-20T08:59:48,192 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-20T08:59:48,592 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=601 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/B/0e11491d25ef497aaeccfdffbd9ada72 2024-11-20T08:59:48,599 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/C/bf2bba36d1bb450e899669c3d182665a is 50, key is test_row_0/C:col10/1732093185611/Put/seqid=0 2024-11-20T08:59:48,604 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073741961_1137 (size=12301) 2024-11-20T08:59:49,005 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=601 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/C/bf2bba36d1bb450e899669c3d182665a 2024-11-20T08:59:49,011 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/A/60c87dc06d614acda9e8c53e10631487 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/60c87dc06d614acda9e8c53e10631487 2024-11-20T08:59:49,016 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/60c87dc06d614acda9e8c53e10631487, entries=150, sequenceid=601, filesize=12.0 K 2024-11-20T08:59:49,017 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/B/0e11491d25ef497aaeccfdffbd9ada72 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/0e11491d25ef497aaeccfdffbd9ada72 2024-11-20T08:59:49,022 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/0e11491d25ef497aaeccfdffbd9ada72, entries=150, sequenceid=601, filesize=12.0 K 2024-11-20T08:59:49,022 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/C/bf2bba36d1bb450e899669c3d182665a as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/C/bf2bba36d1bb450e899669c3d182665a 2024-11-20T08:59:49,026 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/C/bf2bba36d1bb450e899669c3d182665a, entries=150, sequenceid=601, filesize=12.0 K 2024-11-20T08:59:49,027 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=6.71 KB/6870 for 1c52b94a9e6adce80d2d034c31b67cfb in 1258ms, sequenceid=601, compaction requested=false 2024-11-20T08:59:49,027 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1c52b94a9e6adce80d2d034c31b67cfb: 2024-11-20T08:59:55,949 INFO [Thread-143 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-143. 2024-11-20T08:59:55,949 INFO [Thread-149 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-149. 2024-11-20T08:59:55,949 DEBUG [Thread-149 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T08:59:55,949 DEBUG [Thread-143 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T08:59:55,949 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T08:59:55,949 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T08:59:55,949 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T08:59:55,949 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T08:59:55,949 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T08:59:55,949 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T08:59:55,950 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-20T08:59:55,950 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 98 2024-11-20T08:59:55,950 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 106 2024-11-20T08:59:55,950 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 88 2024-11-20T08:59:55,950 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 94 2024-11-20T08:59:55,950 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 92 2024-11-20T08:59:55,950 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-20T08:59:55,950 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2850 2024-11-20T08:59:55,950 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3065 2024-11-20T08:59:55,950 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-20T08:59:55,950 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1384 2024-11-20T08:59:55,950 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4152 rows 2024-11-20T08:59:55,950 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1370 2024-11-20T08:59:55,950 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4104 rows 2024-11-20T08:59:55,951 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-20T08:59:55,951 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMixedAtomicity(AcidGuaranteesTestBase.java:117) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-20T08:59:55,951 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T08:59:55,951 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T08:59:55,951 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T08:59:55,956 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-20T08:59:55,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] procedure2.ProcedureExecutor(1139): Stored pid=23, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-20T08:59:55,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-20T08:59:55,963 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732093195963"}]},"ts":"1732093195963"} 2024-11-20T08:59:55,966 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-20T08:59:55,966 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-20T08:59:55,967 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-20T08:59:55,971 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=25, ppid=24, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1c52b94a9e6adce80d2d034c31b67cfb, UNASSIGN}] 2024-11-20T08:59:55,972 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=25, ppid=24, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1c52b94a9e6adce80d2d034c31b67cfb, UNASSIGN 2024-11-20T08:59:55,973 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=25 updating hbase:meta row=1c52b94a9e6adce80d2d034c31b67cfb, regionState=CLOSING, regionLocation=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:55,975 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=25, ppid=24, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=1c52b94a9e6adce80d2d034c31b67cfb, UNASSIGN because future has completed 2024-11-20T08:59:55,975 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-20T08:59:55,976 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE, hasLock=false; CloseRegionProcedure 1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960}] 2024-11-20T08:59:56,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-20T08:59:56,132 INFO [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] handler.UnassignRegionHandler(122): Close 1c52b94a9e6adce80d2d034c31b67cfb 2024-11-20T08:59:56,132 DEBUG [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-20T08:59:56,133 DEBUG [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1722): Closing 1c52b94a9e6adce80d2d034c31b67cfb, disabling compactions & flushes 2024-11-20T08:59:56,133 INFO [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb. 2024-11-20T08:59:56,133 DEBUG [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb. 2024-11-20T08:59:56,133 DEBUG [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb. after waiting 0 ms 2024-11-20T08:59:56,133 DEBUG [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb. 2024-11-20T08:59:56,133 INFO [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(2902): Flushing 1c52b94a9e6adce80d2d034c31b67cfb 3/3 column families, dataSize=20.13 KB heapSize=53.48 KB 2024-11-20T08:59:56,138 DEBUG [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/A/058cb09e24f94b9c89054f9557503e58 is 50, key is test_row_2/A:col10/1732093195947/Put/seqid=0 2024-11-20T08:59:56,142 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073741962_1138 (size=7415) 2024-11-20T08:59:56,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-20T08:59:56,542 INFO [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=607 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/A/058cb09e24f94b9c89054f9557503e58 2024-11-20T08:59:56,551 DEBUG [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/B/76757a87e59643f582b0ede58466bac9 is 50, key is test_row_2/B:col10/1732093195947/Put/seqid=0 2024-11-20T08:59:56,554 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073741963_1139 (size=7415) 2024-11-20T08:59:56,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-20T08:59:56,955 INFO [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=607 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/B/76757a87e59643f582b0ede58466bac9 2024-11-20T08:59:56,963 DEBUG [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/C/31935c2e6a15454fb91bef984469c283 is 50, key is test_row_2/C:col10/1732093195947/Put/seqid=0 2024-11-20T08:59:56,967 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073741964_1140 (size=7415) 2024-11-20T08:59:57,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-20T08:59:57,368 INFO [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=607 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/C/31935c2e6a15454fb91bef984469c283 2024-11-20T08:59:57,374 DEBUG [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/A/058cb09e24f94b9c89054f9557503e58 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/058cb09e24f94b9c89054f9557503e58 2024-11-20T08:59:57,378 INFO [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/058cb09e24f94b9c89054f9557503e58, entries=50, sequenceid=607, filesize=7.2 K 2024-11-20T08:59:57,379 DEBUG [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/B/76757a87e59643f582b0ede58466bac9 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/76757a87e59643f582b0ede58466bac9 2024-11-20T08:59:57,384 INFO [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/76757a87e59643f582b0ede58466bac9, entries=50, sequenceid=607, filesize=7.2 K 2024-11-20T08:59:57,384 DEBUG [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/.tmp/C/31935c2e6a15454fb91bef984469c283 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/C/31935c2e6a15454fb91bef984469c283 2024-11-20T08:59:57,389 INFO [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/C/31935c2e6a15454fb91bef984469c283, entries=50, sequenceid=607, filesize=7.2 K 2024-11-20T08:59:57,390 INFO [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~53.44 KB/54720, currentSize=0 B/0 for 1c52b94a9e6adce80d2d034c31b67cfb in 1257ms, sequenceid=607, compaction requested=true 2024-11-20T08:59:57,391 DEBUG [StoreCloser-TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/18e402d32c8949b7836f0c78adf42ea6, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/5dad489f7b8e471988ef015ceab18ecf, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/17bb534da4d54370919a658d6e8821fd, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/676c71b52c0c49268ae13273499e7737, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/8a99370765c549529790491b7129bc4b, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/54173cb23d0b4a6a80f35a7c576c08e3, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/b55dc0d1aa924a2491365bda80d65b82, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/6df65a221ea64367a50dbf9cb107dd8a, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/30d99cb2d5a8402887d431d2b8c511f8, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/f947288e99de4974a55d6d8c54a627dc, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/f8265531d4f14044b8a1ad0cac34e7fe, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/20cecef2ca6c43c184c816a58d96b6ba, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/b37f571b76584fbb8c4b6a159dad0bc8, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/658de7bcce954012a0499e1becc615d1, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/29b52998e7714e1dac22d2590d9e48db, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/16ddf979ca0144ca8d520c50c0efd664, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/7c6a5afbecb74501a3c14b41639b7d00, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/feb70ce5513c4de4b8b3ebdf258e8e10, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/8ea1b3fbfc494228bef8f4cf3b493f9b, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/c496b5351b6544e4afadf75baa9478b7, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/9fcdd67484d546e2ae02bbbacc79851b, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/7ab3be557dbf4ae0a380a5d28adf2d18, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/52b72b396afc4518adee8690be75d91b, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/c958825b719a4f51a7e7fb3646106d95, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/e69ecd2769af47a2b1bffe0c9d411e87, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/66966ad6341e413e9e533ab0122d55cb, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/f9fafbb9618c4117977726aa07854ff6, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/4662eb873e1448c78f2efb17a466b9af, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/f1ea035273134700b0f3bfbb548bd814, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/3e40b48d7d93455abc46ce2974f19148, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/b39cf939cd114fa2aff5118b56249034, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/e0beb8f7442a40c48422b266e7d5df63, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/fe9200cfb452410cb92506c681345cba, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/4f6e24e4ae594f2991ae0ecfcaba82d4, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/ae5926da1a5f4f67ba7cca9026fe41b7, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/531c4eb5eb21473287f979f0937cc4fd, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/a08e9bdcb5a94be4b339121b105e1a6f, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/dfbb48b4b3e140319367aa954143dbd8, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/7f4b8131bbd641fbaad7a4a91ab244db, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/55d3a4bb481249be96ff177bf1b85042] to archive 2024-11-20T08:59:57,394 DEBUG [StoreCloser-TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-20T08:59:57,399 DEBUG [StoreCloser-TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/18e402d32c8949b7836f0c78adf42ea6 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/18e402d32c8949b7836f0c78adf42ea6 2024-11-20T08:59:57,401 DEBUG [StoreCloser-TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/5dad489f7b8e471988ef015ceab18ecf to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/5dad489f7b8e471988ef015ceab18ecf 2024-11-20T08:59:57,402 DEBUG [StoreCloser-TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/17bb534da4d54370919a658d6e8821fd to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/17bb534da4d54370919a658d6e8821fd 2024-11-20T08:59:57,403 DEBUG [StoreCloser-TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/676c71b52c0c49268ae13273499e7737 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/676c71b52c0c49268ae13273499e7737 2024-11-20T08:59:57,405 DEBUG [StoreCloser-TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/8a99370765c549529790491b7129bc4b to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/8a99370765c549529790491b7129bc4b 2024-11-20T08:59:57,406 DEBUG [StoreCloser-TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/54173cb23d0b4a6a80f35a7c576c08e3 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/54173cb23d0b4a6a80f35a7c576c08e3 2024-11-20T08:59:57,407 DEBUG [StoreCloser-TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/b55dc0d1aa924a2491365bda80d65b82 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/b55dc0d1aa924a2491365bda80d65b82 2024-11-20T08:59:57,408 DEBUG [StoreCloser-TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/6df65a221ea64367a50dbf9cb107dd8a to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/6df65a221ea64367a50dbf9cb107dd8a 2024-11-20T08:59:57,410 DEBUG [StoreCloser-TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/30d99cb2d5a8402887d431d2b8c511f8 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/30d99cb2d5a8402887d431d2b8c511f8 2024-11-20T08:59:57,411 DEBUG [StoreCloser-TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/f947288e99de4974a55d6d8c54a627dc to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/f947288e99de4974a55d6d8c54a627dc 2024-11-20T08:59:57,412 DEBUG [StoreCloser-TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/f8265531d4f14044b8a1ad0cac34e7fe to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/f8265531d4f14044b8a1ad0cac34e7fe 2024-11-20T08:59:57,413 DEBUG [StoreCloser-TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/20cecef2ca6c43c184c816a58d96b6ba to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/20cecef2ca6c43c184c816a58d96b6ba 2024-11-20T08:59:57,414 DEBUG [StoreCloser-TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/b37f571b76584fbb8c4b6a159dad0bc8 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/b37f571b76584fbb8c4b6a159dad0bc8 2024-11-20T08:59:57,415 DEBUG [StoreCloser-TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/658de7bcce954012a0499e1becc615d1 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/658de7bcce954012a0499e1becc615d1 2024-11-20T08:59:57,417 DEBUG [StoreCloser-TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/29b52998e7714e1dac22d2590d9e48db to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/29b52998e7714e1dac22d2590d9e48db 2024-11-20T08:59:57,418 DEBUG [StoreCloser-TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/16ddf979ca0144ca8d520c50c0efd664 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/16ddf979ca0144ca8d520c50c0efd664 2024-11-20T08:59:57,419 DEBUG [StoreCloser-TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/7c6a5afbecb74501a3c14b41639b7d00 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/7c6a5afbecb74501a3c14b41639b7d00 2024-11-20T08:59:57,420 DEBUG [StoreCloser-TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/feb70ce5513c4de4b8b3ebdf258e8e10 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/feb70ce5513c4de4b8b3ebdf258e8e10 2024-11-20T08:59:57,421 DEBUG [StoreCloser-TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/8ea1b3fbfc494228bef8f4cf3b493f9b to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/8ea1b3fbfc494228bef8f4cf3b493f9b 2024-11-20T08:59:57,422 DEBUG [StoreCloser-TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/c496b5351b6544e4afadf75baa9478b7 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/c496b5351b6544e4afadf75baa9478b7 2024-11-20T08:59:57,424 DEBUG [StoreCloser-TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/9fcdd67484d546e2ae02bbbacc79851b to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/9fcdd67484d546e2ae02bbbacc79851b 2024-11-20T08:59:57,425 DEBUG [StoreCloser-TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/7ab3be557dbf4ae0a380a5d28adf2d18 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/7ab3be557dbf4ae0a380a5d28adf2d18 2024-11-20T08:59:57,426 DEBUG [StoreCloser-TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/52b72b396afc4518adee8690be75d91b to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/52b72b396afc4518adee8690be75d91b 2024-11-20T08:59:57,427 DEBUG [StoreCloser-TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/c958825b719a4f51a7e7fb3646106d95 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/c958825b719a4f51a7e7fb3646106d95 2024-11-20T08:59:57,428 DEBUG [StoreCloser-TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/e69ecd2769af47a2b1bffe0c9d411e87 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/e69ecd2769af47a2b1bffe0c9d411e87 2024-11-20T08:59:57,430 DEBUG [StoreCloser-TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/66966ad6341e413e9e533ab0122d55cb to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/66966ad6341e413e9e533ab0122d55cb 2024-11-20T08:59:57,431 DEBUG [StoreCloser-TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/f9fafbb9618c4117977726aa07854ff6 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/f9fafbb9618c4117977726aa07854ff6 2024-11-20T08:59:57,432 DEBUG [StoreCloser-TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/4662eb873e1448c78f2efb17a466b9af to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/4662eb873e1448c78f2efb17a466b9af 2024-11-20T08:59:57,433 DEBUG [StoreCloser-TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/f1ea035273134700b0f3bfbb548bd814 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/f1ea035273134700b0f3bfbb548bd814 2024-11-20T08:59:57,434 DEBUG [StoreCloser-TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/3e40b48d7d93455abc46ce2974f19148 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/3e40b48d7d93455abc46ce2974f19148 2024-11-20T08:59:57,436 DEBUG [StoreCloser-TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/b39cf939cd114fa2aff5118b56249034 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/b39cf939cd114fa2aff5118b56249034 2024-11-20T08:59:57,437 DEBUG [StoreCloser-TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/e0beb8f7442a40c48422b266e7d5df63 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/e0beb8f7442a40c48422b266e7d5df63 2024-11-20T08:59:57,438 DEBUG [StoreCloser-TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/fe9200cfb452410cb92506c681345cba to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/fe9200cfb452410cb92506c681345cba 2024-11-20T08:59:57,439 DEBUG [StoreCloser-TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/4f6e24e4ae594f2991ae0ecfcaba82d4 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/4f6e24e4ae594f2991ae0ecfcaba82d4 2024-11-20T08:59:57,441 DEBUG [StoreCloser-TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/ae5926da1a5f4f67ba7cca9026fe41b7 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/ae5926da1a5f4f67ba7cca9026fe41b7 2024-11-20T08:59:57,442 DEBUG [StoreCloser-TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/531c4eb5eb21473287f979f0937cc4fd to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/531c4eb5eb21473287f979f0937cc4fd 2024-11-20T08:59:57,443 DEBUG [StoreCloser-TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/a08e9bdcb5a94be4b339121b105e1a6f to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/a08e9bdcb5a94be4b339121b105e1a6f 2024-11-20T08:59:57,445 DEBUG [StoreCloser-TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/dfbb48b4b3e140319367aa954143dbd8 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/dfbb48b4b3e140319367aa954143dbd8 2024-11-20T08:59:57,446 DEBUG [StoreCloser-TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/7f4b8131bbd641fbaad7a4a91ab244db to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/7f4b8131bbd641fbaad7a4a91ab244db 2024-11-20T08:59:57,447 DEBUG [StoreCloser-TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/55d3a4bb481249be96ff177bf1b85042 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/55d3a4bb481249be96ff177bf1b85042 2024-11-20T08:59:57,464 DEBUG [StoreCloser-TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/41c599c3b8ac47f291b46a98be3b0adf, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/03f8e222cd39437caba4b06e0e17c2eb, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/fc87d3e960fe4f3fba71af7835275984, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/a0f5cdf99a1841e6bb7c9d5933220ad1, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/50b9bc81535f4713be9d102b543ca261, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/ccede917c6fb47bf82297b2c25647b36, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/ab247e55e1464dadb2ce6dcc7bbba93e, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/9003c5e71a2e48558bb3a3053d91953b, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/9cbb9795511a463281f5d634616bcbf0, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/81e2c807fc6b4bb892360f41155348a1, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/9799a5642bfe40259b188a37202879cb, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/ac6605a3d8ca4d5ea9bb8abbbadad5d1, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/d47461f002a442bba66ba2b1f5e06318, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/0324d355654845f48a920fdd882fbc25, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/c2af057f4aba4fceb12f7a79ba780730, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/b1b43658f5c84c85b3e2c729ef0dd4f2, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/c1a34a6a5f28497589cce7a85a483073, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/c0351c0dd773448289ffb124f0c87981, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/bf5ed9748a4c4803ab40bc20f78082be, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/80982404f8a746e496f91705020a9fed, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/5f75c8c4487d4d47888476a57f38256b, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/8c563d2983a34d87a1dcc4db311791a6, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/e183aa0a2891486caf4578dc9a002b49, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/3cc6bc716fad4c2fbda70bb12dd2920f, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/51816aaecd03437bb614ac0fd7e0703e, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/c73a991c58c8427685ee4775e0024d86, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/9582a0ce87a041249781a7dfd8a08d47, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/f0e64b206b834513bf0d39333ca33e94, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/dfc65b4a655c47a0a607806d072f14b2, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/3e5841ea82c54ab69aa11e2cd14fb1a9, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/a295cafda18248fd93b17446413328dd, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/51f3985067d2450eb0e0f202b6395189, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/a49c71f381b8482fa40d7e14457556a8, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/a727775cf6e1486e8724e7cedb5341a4, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/04589ae207c7465698a5f1ef95c68f46, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/c9b832fc7a774b949512f1419aa0864a, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/be2c0a116116477a9e070a1ccbf4be2c, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/987c1b45588d47ddbf81c183342babbb, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/e2447008ae3b4e37bd4e0da3466845f9, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/f15f5983f4544ea5a9a2aca942ecc63e] to archive 2024-11-20T08:59:57,465 DEBUG [StoreCloser-TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-20T08:59:57,467 DEBUG [StoreCloser-TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/41c599c3b8ac47f291b46a98be3b0adf to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/41c599c3b8ac47f291b46a98be3b0adf 2024-11-20T08:59:57,468 DEBUG [StoreCloser-TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/03f8e222cd39437caba4b06e0e17c2eb to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/03f8e222cd39437caba4b06e0e17c2eb 2024-11-20T08:59:57,469 DEBUG [StoreCloser-TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/fc87d3e960fe4f3fba71af7835275984 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/fc87d3e960fe4f3fba71af7835275984 2024-11-20T08:59:57,470 DEBUG [StoreCloser-TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/a0f5cdf99a1841e6bb7c9d5933220ad1 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/a0f5cdf99a1841e6bb7c9d5933220ad1 2024-11-20T08:59:57,471 DEBUG [StoreCloser-TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/50b9bc81535f4713be9d102b543ca261 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/50b9bc81535f4713be9d102b543ca261 2024-11-20T08:59:57,473 DEBUG [StoreCloser-TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/ccede917c6fb47bf82297b2c25647b36 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/ccede917c6fb47bf82297b2c25647b36 2024-11-20T08:59:57,474 DEBUG [StoreCloser-TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/ab247e55e1464dadb2ce6dcc7bbba93e to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/ab247e55e1464dadb2ce6dcc7bbba93e 2024-11-20T08:59:57,475 DEBUG [StoreCloser-TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/9003c5e71a2e48558bb3a3053d91953b to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/9003c5e71a2e48558bb3a3053d91953b 2024-11-20T08:59:57,476 DEBUG [StoreCloser-TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/9cbb9795511a463281f5d634616bcbf0 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/9cbb9795511a463281f5d634616bcbf0 2024-11-20T08:59:57,477 DEBUG [StoreCloser-TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/81e2c807fc6b4bb892360f41155348a1 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/81e2c807fc6b4bb892360f41155348a1 2024-11-20T08:59:57,478 DEBUG [StoreCloser-TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/9799a5642bfe40259b188a37202879cb to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/9799a5642bfe40259b188a37202879cb 2024-11-20T08:59:57,479 DEBUG [StoreCloser-TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/ac6605a3d8ca4d5ea9bb8abbbadad5d1 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/ac6605a3d8ca4d5ea9bb8abbbadad5d1 2024-11-20T08:59:57,480 DEBUG [StoreCloser-TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/d47461f002a442bba66ba2b1f5e06318 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/d47461f002a442bba66ba2b1f5e06318 2024-11-20T08:59:57,482 DEBUG [StoreCloser-TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/0324d355654845f48a920fdd882fbc25 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/0324d355654845f48a920fdd882fbc25 2024-11-20T08:59:57,483 DEBUG [StoreCloser-TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/c2af057f4aba4fceb12f7a79ba780730 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/c2af057f4aba4fceb12f7a79ba780730 2024-11-20T08:59:57,484 DEBUG [StoreCloser-TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/b1b43658f5c84c85b3e2c729ef0dd4f2 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/b1b43658f5c84c85b3e2c729ef0dd4f2 2024-11-20T08:59:57,486 DEBUG [StoreCloser-TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/c1a34a6a5f28497589cce7a85a483073 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/c1a34a6a5f28497589cce7a85a483073 2024-11-20T08:59:57,487 DEBUG [StoreCloser-TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/c0351c0dd773448289ffb124f0c87981 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/c0351c0dd773448289ffb124f0c87981 2024-11-20T08:59:57,488 DEBUG [StoreCloser-TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/bf5ed9748a4c4803ab40bc20f78082be to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/bf5ed9748a4c4803ab40bc20f78082be 2024-11-20T08:59:57,489 DEBUG [StoreCloser-TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/80982404f8a746e496f91705020a9fed to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/80982404f8a746e496f91705020a9fed 2024-11-20T08:59:57,490 DEBUG [StoreCloser-TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/5f75c8c4487d4d47888476a57f38256b to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/5f75c8c4487d4d47888476a57f38256b 2024-11-20T08:59:57,491 DEBUG [StoreCloser-TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/8c563d2983a34d87a1dcc4db311791a6 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/8c563d2983a34d87a1dcc4db311791a6 2024-11-20T08:59:57,492 DEBUG [StoreCloser-TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/e183aa0a2891486caf4578dc9a002b49 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/e183aa0a2891486caf4578dc9a002b49 2024-11-20T08:59:57,493 DEBUG [StoreCloser-TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/3cc6bc716fad4c2fbda70bb12dd2920f to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/3cc6bc716fad4c2fbda70bb12dd2920f 2024-11-20T08:59:57,494 DEBUG [StoreCloser-TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/51816aaecd03437bb614ac0fd7e0703e to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/51816aaecd03437bb614ac0fd7e0703e 2024-11-20T08:59:57,496 DEBUG [StoreCloser-TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/c73a991c58c8427685ee4775e0024d86 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/c73a991c58c8427685ee4775e0024d86 2024-11-20T08:59:57,497 DEBUG [StoreCloser-TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/9582a0ce87a041249781a7dfd8a08d47 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/9582a0ce87a041249781a7dfd8a08d47 2024-11-20T08:59:57,498 DEBUG [StoreCloser-TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/f0e64b206b834513bf0d39333ca33e94 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/f0e64b206b834513bf0d39333ca33e94 2024-11-20T08:59:57,499 DEBUG [StoreCloser-TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/dfc65b4a655c47a0a607806d072f14b2 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/dfc65b4a655c47a0a607806d072f14b2 2024-11-20T08:59:57,500 DEBUG [StoreCloser-TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/3e5841ea82c54ab69aa11e2cd14fb1a9 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/3e5841ea82c54ab69aa11e2cd14fb1a9 2024-11-20T08:59:57,501 DEBUG [StoreCloser-TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/a295cafda18248fd93b17446413328dd to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/a295cafda18248fd93b17446413328dd 2024-11-20T08:59:57,502 DEBUG [StoreCloser-TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/51f3985067d2450eb0e0f202b6395189 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/51f3985067d2450eb0e0f202b6395189 2024-11-20T08:59:57,503 DEBUG [StoreCloser-TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/a49c71f381b8482fa40d7e14457556a8 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/a49c71f381b8482fa40d7e14457556a8 2024-11-20T08:59:57,504 DEBUG [StoreCloser-TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/a727775cf6e1486e8724e7cedb5341a4 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/a727775cf6e1486e8724e7cedb5341a4 2024-11-20T08:59:57,505 DEBUG [StoreCloser-TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/04589ae207c7465698a5f1ef95c68f46 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/04589ae207c7465698a5f1ef95c68f46 2024-11-20T08:59:57,507 DEBUG [StoreCloser-TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/c9b832fc7a774b949512f1419aa0864a to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/c9b832fc7a774b949512f1419aa0864a 2024-11-20T08:59:57,508 DEBUG [StoreCloser-TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/be2c0a116116477a9e070a1ccbf4be2c to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/be2c0a116116477a9e070a1ccbf4be2c 2024-11-20T08:59:57,510 DEBUG [StoreCloser-TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/987c1b45588d47ddbf81c183342babbb to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/987c1b45588d47ddbf81c183342babbb 2024-11-20T08:59:57,511 DEBUG [StoreCloser-TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/e2447008ae3b4e37bd4e0da3466845f9 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/e2447008ae3b4e37bd4e0da3466845f9 2024-11-20T08:59:57,512 DEBUG [StoreCloser-TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/f15f5983f4544ea5a9a2aca942ecc63e to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/f15f5983f4544ea5a9a2aca942ecc63e 2024-11-20T08:59:57,515 DEBUG [StoreCloser-TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/C/8af8ce1ea67945a7a89b77ec26594a49, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/C/48fc85d6826d4311b60e6d880cd48696, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/C/9f71a2dc6b5a4d6781eff85497784247, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/C/86e23a1f4fac4b60baca926bfe626ad0, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/C/7b7908ca94e747e9b9eeb1e9db2dc007, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/C/7d1751fdc5ff44bc935c8b93c1d3bb07, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/C/e496a80c051049a08997ed69b0103949, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/C/804fb3a729c04009a44cda091b94774f, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/C/937e0010b140494893f21df2e3750358, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/C/b38ab494a2cf4f8faf649c46a8a08272, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/C/9f5278a2f6b3484ea4ed75021ab2baf8, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/C/b3e41d70621844ba8f3d2ad66c53a171, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/C/cf14c4a783214851a7a97272dc5bc134, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/C/47928dd6881a428d8a2086225812b0ee, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/C/a7229eb625524e8cac1d13ef0ee97a9e, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/C/e7e8076243c74ec1ab9cb9bb438b8166, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/C/a083bcdfec7342809d567f1cb5f5123b, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/C/42168416d26a4307abf7c9dcdbc616bb, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/C/c586a4522ac94f06a72e6ce750175160, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/C/97f6e608690c480fa86d22ec6cf11550, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/C/48e572da1fb345e1b8ac33095a43d011, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/C/18586f39ed8440f29b66f6e7ee1bc1b2, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/C/00759053465245ca936c4cea0d248e6f, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/C/c3426d8981c94354b5e8c062bc6bf12a, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/C/62baecee9b0f4cb1acdb5ba5d32913c5, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/C/5c999896ce04463182cb30a2dabe28e5, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/C/e60b34c184bd43a4b819aa3c0a60cd1c, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/C/0ae3916d365d44a09956735c983b4baa, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/C/4b004e038ded48e585be6e88fd09a463, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/C/6c9cf34eeb4249fd9b7bd4fdb0d6f5c9, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/C/00aeea979ff341178675a0078c5616c2, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/C/54d577d86aca418f91c9ac03730587ec, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/C/bb08f126678947bdb084af69a6eb82b1, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/C/f62fe1d10d6a4fbba8018dcb10f8e72d, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/C/9966d23756d94aaf9e69dd2c5b2c911e, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/C/d4bb1b4b77a848b6b4d2c9cf45aa6601, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/C/1052f6328e014379aeaf32a1555716fb, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/C/ec28ad95e4954444a65fec1c589debfd, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/C/471366db1eb7492f9363719639f52cd9] to archive 2024-11-20T08:59:57,516 DEBUG [StoreCloser-TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-20T08:59:57,517 DEBUG [StoreCloser-TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/C/8af8ce1ea67945a7a89b77ec26594a49 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/C/8af8ce1ea67945a7a89b77ec26594a49 2024-11-20T08:59:57,518 DEBUG [StoreCloser-TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/C/48fc85d6826d4311b60e6d880cd48696 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/C/48fc85d6826d4311b60e6d880cd48696 2024-11-20T08:59:57,520 DEBUG [StoreCloser-TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/C/9f71a2dc6b5a4d6781eff85497784247 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/C/9f71a2dc6b5a4d6781eff85497784247 2024-11-20T08:59:57,521 DEBUG [StoreCloser-TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/C/86e23a1f4fac4b60baca926bfe626ad0 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/C/86e23a1f4fac4b60baca926bfe626ad0 2024-11-20T08:59:57,522 DEBUG [StoreCloser-TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/C/7b7908ca94e747e9b9eeb1e9db2dc007 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/C/7b7908ca94e747e9b9eeb1e9db2dc007 2024-11-20T08:59:57,524 DEBUG [StoreCloser-TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/C/7d1751fdc5ff44bc935c8b93c1d3bb07 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/C/7d1751fdc5ff44bc935c8b93c1d3bb07 2024-11-20T08:59:57,525 DEBUG [StoreCloser-TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/C/e496a80c051049a08997ed69b0103949 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/C/e496a80c051049a08997ed69b0103949 2024-11-20T08:59:57,526 DEBUG [StoreCloser-TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/C/804fb3a729c04009a44cda091b94774f to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/C/804fb3a729c04009a44cda091b94774f 2024-11-20T08:59:57,527 DEBUG [StoreCloser-TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/C/937e0010b140494893f21df2e3750358 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/C/937e0010b140494893f21df2e3750358 2024-11-20T08:59:57,529 DEBUG [StoreCloser-TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/C/b38ab494a2cf4f8faf649c46a8a08272 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/C/b38ab494a2cf4f8faf649c46a8a08272 2024-11-20T08:59:57,530 DEBUG [StoreCloser-TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/C/9f5278a2f6b3484ea4ed75021ab2baf8 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/C/9f5278a2f6b3484ea4ed75021ab2baf8 2024-11-20T08:59:57,531 DEBUG [StoreCloser-TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/C/b3e41d70621844ba8f3d2ad66c53a171 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/C/b3e41d70621844ba8f3d2ad66c53a171 2024-11-20T08:59:57,532 DEBUG [StoreCloser-TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/C/cf14c4a783214851a7a97272dc5bc134 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/C/cf14c4a783214851a7a97272dc5bc134 2024-11-20T08:59:57,534 DEBUG [StoreCloser-TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/C/47928dd6881a428d8a2086225812b0ee to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/C/47928dd6881a428d8a2086225812b0ee 2024-11-20T08:59:57,535 DEBUG [StoreCloser-TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/C/a7229eb625524e8cac1d13ef0ee97a9e to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/C/a7229eb625524e8cac1d13ef0ee97a9e 2024-11-20T08:59:57,536 DEBUG [StoreCloser-TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/C/e7e8076243c74ec1ab9cb9bb438b8166 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/C/e7e8076243c74ec1ab9cb9bb438b8166 2024-11-20T08:59:57,537 DEBUG [StoreCloser-TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/C/a083bcdfec7342809d567f1cb5f5123b to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/C/a083bcdfec7342809d567f1cb5f5123b 2024-11-20T08:59:57,538 DEBUG [StoreCloser-TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/C/42168416d26a4307abf7c9dcdbc616bb to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/C/42168416d26a4307abf7c9dcdbc616bb 2024-11-20T08:59:57,539 DEBUG [StoreCloser-TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/C/c586a4522ac94f06a72e6ce750175160 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/C/c586a4522ac94f06a72e6ce750175160 2024-11-20T08:59:57,541 DEBUG [StoreCloser-TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/C/97f6e608690c480fa86d22ec6cf11550 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/C/97f6e608690c480fa86d22ec6cf11550 2024-11-20T08:59:57,542 DEBUG [StoreCloser-TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/C/48e572da1fb345e1b8ac33095a43d011 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/C/48e572da1fb345e1b8ac33095a43d011 2024-11-20T08:59:57,543 DEBUG [StoreCloser-TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/C/18586f39ed8440f29b66f6e7ee1bc1b2 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/C/18586f39ed8440f29b66f6e7ee1bc1b2 2024-11-20T08:59:57,544 DEBUG [StoreCloser-TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/C/00759053465245ca936c4cea0d248e6f to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/C/00759053465245ca936c4cea0d248e6f 2024-11-20T08:59:57,545 DEBUG [StoreCloser-TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/C/c3426d8981c94354b5e8c062bc6bf12a to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/C/c3426d8981c94354b5e8c062bc6bf12a 2024-11-20T08:59:57,546 DEBUG [StoreCloser-TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/C/62baecee9b0f4cb1acdb5ba5d32913c5 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/C/62baecee9b0f4cb1acdb5ba5d32913c5 2024-11-20T08:59:57,548 DEBUG [StoreCloser-TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/C/5c999896ce04463182cb30a2dabe28e5 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/C/5c999896ce04463182cb30a2dabe28e5 2024-11-20T08:59:57,549 DEBUG [StoreCloser-TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/C/e60b34c184bd43a4b819aa3c0a60cd1c to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/C/e60b34c184bd43a4b819aa3c0a60cd1c 2024-11-20T08:59:57,551 DEBUG [StoreCloser-TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/C/0ae3916d365d44a09956735c983b4baa to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/C/0ae3916d365d44a09956735c983b4baa 2024-11-20T08:59:57,552 DEBUG [StoreCloser-TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/C/4b004e038ded48e585be6e88fd09a463 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/C/4b004e038ded48e585be6e88fd09a463 2024-11-20T08:59:57,553 DEBUG [StoreCloser-TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/C/6c9cf34eeb4249fd9b7bd4fdb0d6f5c9 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/C/6c9cf34eeb4249fd9b7bd4fdb0d6f5c9 2024-11-20T08:59:57,554 DEBUG [StoreCloser-TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/C/00aeea979ff341178675a0078c5616c2 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/C/00aeea979ff341178675a0078c5616c2 2024-11-20T08:59:57,555 DEBUG [StoreCloser-TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/C/54d577d86aca418f91c9ac03730587ec to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/C/54d577d86aca418f91c9ac03730587ec 2024-11-20T08:59:57,557 DEBUG [StoreCloser-TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/C/bb08f126678947bdb084af69a6eb82b1 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/C/bb08f126678947bdb084af69a6eb82b1 2024-11-20T08:59:57,558 DEBUG [StoreCloser-TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/C/f62fe1d10d6a4fbba8018dcb10f8e72d to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/C/f62fe1d10d6a4fbba8018dcb10f8e72d 2024-11-20T08:59:57,559 DEBUG [StoreCloser-TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/C/9966d23756d94aaf9e69dd2c5b2c911e to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/C/9966d23756d94aaf9e69dd2c5b2c911e 2024-11-20T08:59:57,560 DEBUG [StoreCloser-TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/C/d4bb1b4b77a848b6b4d2c9cf45aa6601 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/C/d4bb1b4b77a848b6b4d2c9cf45aa6601 2024-11-20T08:59:57,562 DEBUG [StoreCloser-TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/C/1052f6328e014379aeaf32a1555716fb to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/C/1052f6328e014379aeaf32a1555716fb 2024-11-20T08:59:57,563 DEBUG [StoreCloser-TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/C/ec28ad95e4954444a65fec1c589debfd to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/C/ec28ad95e4954444a65fec1c589debfd 2024-11-20T08:59:57,564 DEBUG [StoreCloser-TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/C/471366db1eb7492f9363719639f52cd9 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/C/471366db1eb7492f9363719639f52cd9 2024-11-20T08:59:57,569 DEBUG [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/recovered.edits/610.seqid, newMaxSeqId=610, maxSeqId=1 2024-11-20T08:59:57,570 INFO [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb. 2024-11-20T08:59:57,570 DEBUG [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1676): Region close journal for 1c52b94a9e6adce80d2d034c31b67cfb: Waiting for close lock at 1732093196133Running coprocessor pre-close hooks at 1732093196133Disabling compacts and flushes for region at 1732093196133Disabling writes for close at 1732093196133Obtaining lock to block concurrent updates at 1732093196133Preparing flush snapshotting stores in 1c52b94a9e6adce80d2d034c31b67cfb at 1732093196133Finished memstore snapshotting TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb., syncing WAL and waiting on mvcc, flushsize=dataSize=20610, getHeapSize=54720, getOffHeapSize=0, getCellsCount=450 at 1732093196134 (+1 ms)Flushing stores of TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb. at 1732093196134Flushing 1c52b94a9e6adce80d2d034c31b67cfb/A: creating writer at 1732093196134Flushing 1c52b94a9e6adce80d2d034c31b67cfb/A: appending metadata at 1732093196137 (+3 ms)Flushing 1c52b94a9e6adce80d2d034c31b67cfb/A: closing flushed file at 1732093196137Flushing 1c52b94a9e6adce80d2d034c31b67cfb/B: creating writer at 1732093196548 (+411 ms)Flushing 1c52b94a9e6adce80d2d034c31b67cfb/B: appending metadata at 1732093196550 (+2 ms)Flushing 1c52b94a9e6adce80d2d034c31b67cfb/B: closing flushed file at 1732093196550Flushing 1c52b94a9e6adce80d2d034c31b67cfb/C: creating writer at 1732093196960 (+410 ms)Flushing 1c52b94a9e6adce80d2d034c31b67cfb/C: appending metadata at 1732093196963 (+3 ms)Flushing 1c52b94a9e6adce80d2d034c31b67cfb/C: closing flushed file at 1732093196963Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7944f148: reopening flushed file at 1732093197373 (+410 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6e34c73f: reopening flushed file at 1732093197379 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1c1dd5f3: reopening flushed file at 1732093197384 (+5 ms)Finished flush of dataSize ~20.13 KB/20610, heapSize ~53.44 KB/54720, currentSize=0 B/0 for 1c52b94a9e6adce80d2d034c31b67cfb in 1257ms, sequenceid=607, compaction requested=true at 1732093197390 (+6 ms)Writing region close event to WAL at 1732093197566 (+176 ms)Running coprocessor post-close hooks at 1732093197570 (+4 ms)Closed at 1732093197570 2024-11-20T08:59:57,573 INFO [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] handler.UnassignRegionHandler(157): Closed 1c52b94a9e6adce80d2d034c31b67cfb 2024-11-20T08:59:57,573 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=25 updating hbase:meta row=1c52b94a9e6adce80d2d034c31b67cfb, regionState=CLOSED 2024-11-20T08:59:57,575 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=26, ppid=25, state=RUNNABLE, hasLock=false; CloseRegionProcedure 1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 because future has completed 2024-11-20T08:59:57,578 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-11-20T08:59:57,578 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; CloseRegionProcedure 1c52b94a9e6adce80d2d034c31b67cfb, server=be1eb2620e0e,34223,1732093159960 in 1.6000 sec 2024-11-20T08:59:57,579 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=25, resume processing ppid=24 2024-11-20T08:59:57,579 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, ppid=24, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1c52b94a9e6adce80d2d034c31b67cfb, UNASSIGN in 1.6070 sec 2024-11-20T08:59:57,582 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-11-20T08:59:57,582 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.6130 sec 2024-11-20T08:59:57,583 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732093197583"}]},"ts":"1732093197583"} 2024-11-20T08:59:57,584 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-20T08:59:57,584 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-20T08:59:57,586 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.6270 sec 2024-11-20T08:59:58,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-20T08:59:58,108 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-20T08:59:58,111 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-20T08:59:58,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] procedure2.ProcedureExecutor(1139): Stored pid=27, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T08:59:58,116 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=27, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T08:59:58,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-20T08:59:58,117 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=27, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T08:59:58,122 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb 2024-11-20T08:59:58,126 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A, FileablePath, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B, FileablePath, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/C, FileablePath, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/recovered.edits] 2024-11-20T08:59:58,129 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/058cb09e24f94b9c89054f9557503e58 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/058cb09e24f94b9c89054f9557503e58 2024-11-20T08:59:58,130 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/243f2dc7d8b6433bb1658703919b6b00 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/243f2dc7d8b6433bb1658703919b6b00 2024-11-20T08:59:58,131 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/60c87dc06d614acda9e8c53e10631487 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/A/60c87dc06d614acda9e8c53e10631487 2024-11-20T08:59:58,134 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/0e11491d25ef497aaeccfdffbd9ada72 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/0e11491d25ef497aaeccfdffbd9ada72 2024-11-20T08:59:58,135 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/76757a87e59643f582b0ede58466bac9 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/76757a87e59643f582b0ede58466bac9 2024-11-20T08:59:58,137 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/8018f60f25be47dbbeb0e69ea98d36b7 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/B/8018f60f25be47dbbeb0e69ea98d36b7 2024-11-20T08:59:58,139 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/C/31935c2e6a15454fb91bef984469c283 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/C/31935c2e6a15454fb91bef984469c283 2024-11-20T08:59:58,141 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/C/bf2bba36d1bb450e899669c3d182665a to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/C/bf2bba36d1bb450e899669c3d182665a 2024-11-20T08:59:58,142 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/C/e0cc30253b754f00a46aa1ae26b9443a to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/C/e0cc30253b754f00a46aa1ae26b9443a 2024-11-20T08:59:58,145 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/recovered.edits/610.seqid to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb/recovered.edits/610.seqid 2024-11-20T08:59:58,146 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/1c52b94a9e6adce80d2d034c31b67cfb 2024-11-20T08:59:58,146 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-20T08:59:58,151 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=27, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T08:59:58,155 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34223 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-11-20T08:59:58,160 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-20T08:59:58,190 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-20T08:59:58,192 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=27, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T08:59:58,192 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-20T08:59:58,192 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732093198192"}]},"ts":"9223372036854775807"} 2024-11-20T08:59:58,196 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-20T08:59:58,196 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 1c52b94a9e6adce80d2d034c31b67cfb, NAME => 'TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb.', STARTKEY => '', ENDKEY => ''}] 2024-11-20T08:59:58,196 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-20T08:59:58,197 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732093198196"}]},"ts":"9223372036854775807"} 2024-11-20T08:59:58,199 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-20T08:59:58,200 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=27, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T08:59:58,201 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 89 msec 2024-11-20T08:59:58,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-20T08:59:58,228 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-20T08:59:58,228 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-20T08:59:58,243 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMixedAtomicity Thread=236 (was 210) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS:0;be1eb2620e0e:34223-shortCompactions-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:535) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_656394272_22 at /127.0.0.1:53848 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=449 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=381 (was 257) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 12), AvailableMemoryMB=2376 (was 2930) 2024-11-20T08:59:58,254 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobMixedAtomicity Thread=236, OpenFileDescriptor=449, MaxFileDescriptor=1048576, SystemLoadAverage=381, ProcessCount=11, AvailableMemoryMB=2372 2024-11-20T08:59:58,255 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-20T08:59:58,256 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-20T08:59:58,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] procedure2.ProcedureExecutor(1139): Stored pid=28, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-20T08:59:58,258 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-20T08:59:58,258 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T08:59:58,258 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 28 2024-11-20T08:59:58,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-11-20T08:59:58,259 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-20T08:59:58,265 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073741965_1141 (size=959) 2024-11-20T08:59:58,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-11-20T08:59:58,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-11-20T08:59:58,667 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => f0e725d6c797c04eb86dfc3be87ab3ba, NAME => 'TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a 2024-11-20T08:59:58,672 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073741966_1142 (size=53) 2024-11-20T08:59:58,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-11-20T08:59:59,074 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T08:59:59,074 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing f0e725d6c797c04eb86dfc3be87ab3ba, disabling compactions & flushes 2024-11-20T08:59:59,074 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. 2024-11-20T08:59:59,074 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. 2024-11-20T08:59:59,074 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. after waiting 0 ms 2024-11-20T08:59:59,074 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. 2024-11-20T08:59:59,074 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. 2024-11-20T08:59:59,074 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for f0e725d6c797c04eb86dfc3be87ab3ba: Waiting for close lock at 1732093199074Disabling compacts and flushes for region at 1732093199074Disabling writes for close at 1732093199074Writing region close event to WAL at 1732093199074Closed at 1732093199074 2024-11-20T08:59:59,075 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-20T08:59:59,076 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732093199075"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732093199075"}]},"ts":"1732093199075"} 2024-11-20T08:59:59,078 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-20T08:59:59,079 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-20T08:59:59,079 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732093199079"}]},"ts":"1732093199079"} 2024-11-20T08:59:59,080 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-20T08:59:59,081 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f0e725d6c797c04eb86dfc3be87ab3ba, ASSIGN}] 2024-11-20T08:59:59,082 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f0e725d6c797c04eb86dfc3be87ab3ba, ASSIGN 2024-11-20T08:59:59,083 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f0e725d6c797c04eb86dfc3be87ab3ba, ASSIGN; state=OFFLINE, location=be1eb2620e0e,34223,1732093159960; forceNewPlan=false, retain=false 2024-11-20T08:59:59,233 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=f0e725d6c797c04eb86dfc3be87ab3ba, regionState=OPENING, regionLocation=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:59,235 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f0e725d6c797c04eb86dfc3be87ab3ba, ASSIGN because future has completed 2024-11-20T08:59:59,236 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960}] 2024-11-20T08:59:59,390 INFO [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. 2024-11-20T08:59:59,391 DEBUG [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(7752): Opening region: {ENCODED => f0e725d6c797c04eb86dfc3be87ab3ba, NAME => 'TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.', STARTKEY => '', ENDKEY => ''} 2024-11-20T08:59:59,391 DEBUG [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees f0e725d6c797c04eb86dfc3be87ab3ba 2024-11-20T08:59:59,391 DEBUG [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T08:59:59,391 DEBUG [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(7794): checking encryption for f0e725d6c797c04eb86dfc3be87ab3ba 2024-11-20T08:59:59,391 DEBUG [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(7797): checking classloading for f0e725d6c797c04eb86dfc3be87ab3ba 2024-11-20T08:59:59,393 INFO [StoreOpener-f0e725d6c797c04eb86dfc3be87ab3ba-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region f0e725d6c797c04eb86dfc3be87ab3ba 2024-11-20T08:59:59,395 INFO [StoreOpener-f0e725d6c797c04eb86dfc3be87ab3ba-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f0e725d6c797c04eb86dfc3be87ab3ba columnFamilyName A 2024-11-20T08:59:59,395 DEBUG [StoreOpener-f0e725d6c797c04eb86dfc3be87ab3ba-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T08:59:59,396 INFO [StoreOpener-f0e725d6c797c04eb86dfc3be87ab3ba-1 {}] regionserver.HStore(327): Store=f0e725d6c797c04eb86dfc3be87ab3ba/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T08:59:59,396 INFO [StoreOpener-f0e725d6c797c04eb86dfc3be87ab3ba-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region f0e725d6c797c04eb86dfc3be87ab3ba 2024-11-20T08:59:59,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-11-20T08:59:59,398 INFO [StoreOpener-f0e725d6c797c04eb86dfc3be87ab3ba-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f0e725d6c797c04eb86dfc3be87ab3ba columnFamilyName B 2024-11-20T08:59:59,398 DEBUG [StoreOpener-f0e725d6c797c04eb86dfc3be87ab3ba-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T08:59:59,398 INFO [StoreOpener-f0e725d6c797c04eb86dfc3be87ab3ba-1 {}] regionserver.HStore(327): Store=f0e725d6c797c04eb86dfc3be87ab3ba/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T08:59:59,398 INFO [StoreOpener-f0e725d6c797c04eb86dfc3be87ab3ba-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region f0e725d6c797c04eb86dfc3be87ab3ba 2024-11-20T08:59:59,400 INFO [StoreOpener-f0e725d6c797c04eb86dfc3be87ab3ba-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f0e725d6c797c04eb86dfc3be87ab3ba columnFamilyName C 2024-11-20T08:59:59,400 DEBUG [StoreOpener-f0e725d6c797c04eb86dfc3be87ab3ba-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T08:59:59,400 INFO [StoreOpener-f0e725d6c797c04eb86dfc3be87ab3ba-1 {}] regionserver.HStore(327): Store=f0e725d6c797c04eb86dfc3be87ab3ba/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T08:59:59,400 DEBUG [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1038): replaying wal for f0e725d6c797c04eb86dfc3be87ab3ba 2024-11-20T08:59:59,401 DEBUG [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba 2024-11-20T08:59:59,401 DEBUG [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba 2024-11-20T08:59:59,402 DEBUG [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1048): stopping wal replay for f0e725d6c797c04eb86dfc3be87ab3ba 2024-11-20T08:59:59,402 DEBUG [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1060): Cleaning up temporary data for f0e725d6c797c04eb86dfc3be87ab3ba 2024-11-20T08:59:59,403 DEBUG [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-20T08:59:59,404 DEBUG [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1093): writing seq id for f0e725d6c797c04eb86dfc3be87ab3ba 2024-11-20T08:59:59,406 DEBUG [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T08:59:59,406 INFO [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1114): Opened f0e725d6c797c04eb86dfc3be87ab3ba; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61844720, jitterRate=-0.07844185829162598}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-20T08:59:59,406 DEBUG [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f0e725d6c797c04eb86dfc3be87ab3ba 2024-11-20T08:59:59,407 DEBUG [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1006): Region open journal for f0e725d6c797c04eb86dfc3be87ab3ba: Running coprocessor pre-open hook at 1732093199392Writing region info on filesystem at 1732093199392Initializing all the Stores at 1732093199392Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732093199392Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732093199393 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732093199393Cleaning up temporary data from old regions at 1732093199402 (+9 ms)Running coprocessor post-open hooks at 1732093199406 (+4 ms)Region opened successfully at 1732093199407 (+1 ms) 2024-11-20T08:59:59,408 INFO [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., pid=30, masterSystemTime=1732093199387 2024-11-20T08:59:59,411 DEBUG [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. 2024-11-20T08:59:59,411 INFO [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. 2024-11-20T08:59:59,411 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=f0e725d6c797c04eb86dfc3be87ab3ba, regionState=OPEN, openSeqNum=2, regionLocation=be1eb2620e0e,34223,1732093159960 2024-11-20T08:59:59,413 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=30, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 because future has completed 2024-11-20T08:59:59,416 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29 2024-11-20T08:59:59,416 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; OpenRegionProcedure f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 in 178 msec 2024-11-20T08:59:59,418 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=29, resume processing ppid=28 2024-11-20T08:59:59,418 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, ppid=28, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f0e725d6c797c04eb86dfc3be87ab3ba, ASSIGN in 335 msec 2024-11-20T08:59:59,419 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-20T08:59:59,419 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732093199419"}]},"ts":"1732093199419"} 2024-11-20T08:59:59,420 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-20T08:59:59,421 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-20T08:59:59,422 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1650 sec 2024-11-20T09:00:00,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-11-20T09:00:00,407 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-20T09:00:00,408 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@338737eb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T09:00:00,408 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request be1eb2620e0e,44373,-1 for getting cluster id 2024-11-20T09:00:00,409 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T09:00:00,410 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'caf50035-b19e-4cfd-90ea-dca559866882' 2024-11-20T09:00:00,410 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T09:00:00,411 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "caf50035-b19e-4cfd-90ea-dca559866882" 2024-11-20T09:00:00,411 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2a33536f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T09:00:00,411 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [be1eb2620e0e,44373,-1] 2024-11-20T09:00:00,411 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T09:00:00,411 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T09:00:00,412 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60890, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T09:00:00,413 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@121de19e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T09:00:00,413 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T09:00:00,414 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=be1eb2620e0e,34223,1732093159960, seqNum=-1] 2024-11-20T09:00:00,415 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T09:00:00,416 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48374, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T09:00:00,417 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-20T09:00:00,418 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is be1eb2620e0e,44373,1732093159150 2024-11-20T09:00:00,418 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@c90b79a 2024-11-20T09:00:00,418 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-20T09:00:00,419 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60904, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-20T09:00:00,424 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-20T09:00:00,424 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-20T09:00:00,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] procedure2.ProcedureExecutor(1139): Stored pid=31, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-20T09:00:00,444 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073741967_1143 (size=995) 2024-11-20T09:00:00,846 DEBUG [PEWorker-4 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-11-20T09:00:00,846 INFO [PEWorker-4 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-11-20T09:00:00,850 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-20T09:00:00,857 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f0e725d6c797c04eb86dfc3be87ab3ba, REOPEN/MOVE}] 2024-11-20T09:00:00,858 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f0e725d6c797c04eb86dfc3be87ab3ba, REOPEN/MOVE 2024-11-20T09:00:00,858 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=f0e725d6c797c04eb86dfc3be87ab3ba, regionState=CLOSING, regionLocation=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:00,860 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f0e725d6c797c04eb86dfc3be87ab3ba, REOPEN/MOVE because future has completed 2024-11-20T09:00:00,861 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-20T09:00:00,861 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=34, ppid=33, state=RUNNABLE, hasLock=false; CloseRegionProcedure f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960}] 2024-11-20T09:00:01,013 INFO [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] handler.UnassignRegionHandler(122): Close f0e725d6c797c04eb86dfc3be87ab3ba 2024-11-20T09:00:01,013 DEBUG [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-20T09:00:01,013 DEBUG [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1722): Closing f0e725d6c797c04eb86dfc3be87ab3ba, disabling compactions & flushes 2024-11-20T09:00:01,013 INFO [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. 2024-11-20T09:00:01,013 DEBUG [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. 2024-11-20T09:00:01,013 DEBUG [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. after waiting 0 ms 2024-11-20T09:00:01,013 DEBUG [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. 2024-11-20T09:00:01,017 DEBUG [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-20T09:00:01,018 INFO [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. 2024-11-20T09:00:01,018 DEBUG [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1676): Region close journal for f0e725d6c797c04eb86dfc3be87ab3ba: Waiting for close lock at 1732093201013Running coprocessor pre-close hooks at 1732093201013Disabling compacts and flushes for region at 1732093201013Disabling writes for close at 1732093201013Writing region close event to WAL at 1732093201014 (+1 ms)Running coprocessor post-close hooks at 1732093201018 (+4 ms)Closed at 1732093201018 2024-11-20T09:00:01,018 WARN [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegionServer(3299): Not adding moved region record: f0e725d6c797c04eb86dfc3be87ab3ba to self. 2024-11-20T09:00:01,020 INFO [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] handler.UnassignRegionHandler(157): Closed f0e725d6c797c04eb86dfc3be87ab3ba 2024-11-20T09:00:01,021 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=f0e725d6c797c04eb86dfc3be87ab3ba, regionState=CLOSED 2024-11-20T09:00:01,022 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=34, ppid=33, state=RUNNABLE, hasLock=false; CloseRegionProcedure f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 because future has completed 2024-11-20T09:00:01,025 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=34, resume processing ppid=33 2024-11-20T09:00:01,025 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, ppid=33, state=SUCCESS, hasLock=false; CloseRegionProcedure f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 in 162 msec 2024-11-20T09:00:01,026 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f0e725d6c797c04eb86dfc3be87ab3ba, REOPEN/MOVE; state=CLOSED, location=be1eb2620e0e,34223,1732093159960; forceNewPlan=false, retain=true 2024-11-20T09:00:01,176 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=f0e725d6c797c04eb86dfc3be87ab3ba, regionState=OPENING, regionLocation=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:01,178 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f0e725d6c797c04eb86dfc3be87ab3ba, REOPEN/MOVE because future has completed 2024-11-20T09:00:01,178 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=35, ppid=33, state=RUNNABLE, hasLock=false; OpenRegionProcedure f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960}] 2024-11-20T09:00:01,334 INFO [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. 2024-11-20T09:00:01,334 DEBUG [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7752): Opening region: {ENCODED => f0e725d6c797c04eb86dfc3be87ab3ba, NAME => 'TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.', STARTKEY => '', ENDKEY => ''} 2024-11-20T09:00:01,334 DEBUG [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees f0e725d6c797c04eb86dfc3be87ab3ba 2024-11-20T09:00:01,334 DEBUG [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T09:00:01,334 DEBUG [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7794): checking encryption for f0e725d6c797c04eb86dfc3be87ab3ba 2024-11-20T09:00:01,334 DEBUG [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7797): checking classloading for f0e725d6c797c04eb86dfc3be87ab3ba 2024-11-20T09:00:01,339 INFO [StoreOpener-f0e725d6c797c04eb86dfc3be87ab3ba-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region f0e725d6c797c04eb86dfc3be87ab3ba 2024-11-20T09:00:01,345 INFO [StoreOpener-f0e725d6c797c04eb86dfc3be87ab3ba-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f0e725d6c797c04eb86dfc3be87ab3ba columnFamilyName A 2024-11-20T09:00:01,347 DEBUG [StoreOpener-f0e725d6c797c04eb86dfc3be87ab3ba-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:01,347 INFO [StoreOpener-f0e725d6c797c04eb86dfc3be87ab3ba-1 {}] regionserver.HStore(327): Store=f0e725d6c797c04eb86dfc3be87ab3ba/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T09:00:01,348 INFO [StoreOpener-f0e725d6c797c04eb86dfc3be87ab3ba-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region f0e725d6c797c04eb86dfc3be87ab3ba 2024-11-20T09:00:01,348 INFO [StoreOpener-f0e725d6c797c04eb86dfc3be87ab3ba-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f0e725d6c797c04eb86dfc3be87ab3ba columnFamilyName B 2024-11-20T09:00:01,348 DEBUG [StoreOpener-f0e725d6c797c04eb86dfc3be87ab3ba-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:01,349 INFO [StoreOpener-f0e725d6c797c04eb86dfc3be87ab3ba-1 {}] regionserver.HStore(327): Store=f0e725d6c797c04eb86dfc3be87ab3ba/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T09:00:01,349 INFO [StoreOpener-f0e725d6c797c04eb86dfc3be87ab3ba-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region f0e725d6c797c04eb86dfc3be87ab3ba 2024-11-20T09:00:01,350 INFO [StoreOpener-f0e725d6c797c04eb86dfc3be87ab3ba-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f0e725d6c797c04eb86dfc3be87ab3ba columnFamilyName C 2024-11-20T09:00:01,350 DEBUG [StoreOpener-f0e725d6c797c04eb86dfc3be87ab3ba-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:01,350 INFO [StoreOpener-f0e725d6c797c04eb86dfc3be87ab3ba-1 {}] regionserver.HStore(327): Store=f0e725d6c797c04eb86dfc3be87ab3ba/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T09:00:01,350 DEBUG [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1038): replaying wal for f0e725d6c797c04eb86dfc3be87ab3ba 2024-11-20T09:00:01,351 DEBUG [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba 2024-11-20T09:00:01,352 DEBUG [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba 2024-11-20T09:00:01,353 DEBUG [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1048): stopping wal replay for f0e725d6c797c04eb86dfc3be87ab3ba 2024-11-20T09:00:01,353 DEBUG [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1060): Cleaning up temporary data for f0e725d6c797c04eb86dfc3be87ab3ba 2024-11-20T09:00:01,353 DEBUG [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-20T09:00:01,354 DEBUG [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1093): writing seq id for f0e725d6c797c04eb86dfc3be87ab3ba 2024-11-20T09:00:01,355 INFO [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1114): Opened f0e725d6c797c04eb86dfc3be87ab3ba; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64391141, jitterRate=-0.040497228503227234}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-20T09:00:01,355 DEBUG [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f0e725d6c797c04eb86dfc3be87ab3ba 2024-11-20T09:00:01,356 DEBUG [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1006): Region open journal for f0e725d6c797c04eb86dfc3be87ab3ba: Running coprocessor pre-open hook at 1732093201335Writing region info on filesystem at 1732093201335Initializing all the Stores at 1732093201335Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732093201335Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732093201336 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732093201336Cleaning up temporary data from old regions at 1732093201353 (+17 ms)Running coprocessor post-open hooks at 1732093201355 (+2 ms)Region opened successfully at 1732093201356 (+1 ms) 2024-11-20T09:00:01,357 INFO [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., pid=35, masterSystemTime=1732093201330 2024-11-20T09:00:01,359 DEBUG [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. 2024-11-20T09:00:01,359 INFO [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. 2024-11-20T09:00:01,359 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=f0e725d6c797c04eb86dfc3be87ab3ba, regionState=OPEN, openSeqNum=5, regionLocation=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:01,361 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=33, state=RUNNABLE, hasLock=false; OpenRegionProcedure f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 because future has completed 2024-11-20T09:00:01,363 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=35, resume processing ppid=33 2024-11-20T09:00:01,363 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=33, state=SUCCESS, hasLock=false; OpenRegionProcedure f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 in 184 msec 2024-11-20T09:00:01,365 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=32 2024-11-20T09:00:01,365 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f0e725d6c797c04eb86dfc3be87ab3ba, REOPEN/MOVE in 506 msec 2024-11-20T09:00:01,368 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=31 2024-11-20T09:00:01,368 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=31, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 517 msec 2024-11-20T09:00:01,371 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 942 msec 2024-11-20T09:00:01,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-20T09:00:01,373 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-20T09:00:01,374 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5bb0fd71, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T09:00:01,374 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request be1eb2620e0e,44373,-1 for getting cluster id 2024-11-20T09:00:01,374 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T09:00:01,375 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'caf50035-b19e-4cfd-90ea-dca559866882' 2024-11-20T09:00:01,375 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T09:00:01,375 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "caf50035-b19e-4cfd-90ea-dca559866882" 2024-11-20T09:00:01,375 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1074e8e5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T09:00:01,375 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [be1eb2620e0e,44373,-1] 2024-11-20T09:00:01,376 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T09:00:01,376 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T09:00:01,377 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60922, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T09:00:01,377 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e1745a4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T09:00:01,378 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68ed5d71, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T09:00:01,378 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request be1eb2620e0e,44373,-1 for getting cluster id 2024-11-20T09:00:01,379 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T09:00:01,379 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'caf50035-b19e-4cfd-90ea-dca559866882' 2024-11-20T09:00:01,379 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T09:00:01,380 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "caf50035-b19e-4cfd-90ea-dca559866882" 2024-11-20T09:00:01,380 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1dbb8645, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T09:00:01,380 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [be1eb2620e0e,44373,-1] 2024-11-20T09:00:01,380 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T09:00:01,380 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T09:00:01,381 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60936, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T09:00:01,381 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1e4a8b0f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T09:00:01,382 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4406a4b8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T09:00:01,382 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request be1eb2620e0e,44373,-1 for getting cluster id 2024-11-20T09:00:01,382 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T09:00:01,383 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'caf50035-b19e-4cfd-90ea-dca559866882' 2024-11-20T09:00:01,383 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T09:00:01,383 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "caf50035-b19e-4cfd-90ea-dca559866882" 2024-11-20T09:00:01,384 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@30bb896e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T09:00:01,384 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [be1eb2620e0e,44373,-1] 2024-11-20T09:00:01,384 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T09:00:01,384 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T09:00:01,385 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60944, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T09:00:01,385 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@383b693c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T09:00:01,386 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7230d245, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T09:00:01,386 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request be1eb2620e0e,44373,-1 for getting cluster id 2024-11-20T09:00:01,387 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T09:00:01,388 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'caf50035-b19e-4cfd-90ea-dca559866882' 2024-11-20T09:00:01,388 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T09:00:01,388 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "caf50035-b19e-4cfd-90ea-dca559866882" 2024-11-20T09:00:01,388 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c253f67, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T09:00:01,388 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [be1eb2620e0e,44373,-1] 2024-11-20T09:00:01,388 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T09:00:01,388 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T09:00:01,389 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60956, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T09:00:01,390 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b9457b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T09:00:01,390 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c77b203, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T09:00:01,390 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request be1eb2620e0e,44373,-1 for getting cluster id 2024-11-20T09:00:01,391 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T09:00:01,391 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'caf50035-b19e-4cfd-90ea-dca559866882' 2024-11-20T09:00:01,391 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T09:00:01,391 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "caf50035-b19e-4cfd-90ea-dca559866882" 2024-11-20T09:00:01,392 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1f690efc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T09:00:01,392 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [be1eb2620e0e,44373,-1] 2024-11-20T09:00:01,392 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T09:00:01,392 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T09:00:01,393 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60978, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T09:00:01,393 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b6c5d96, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T09:00:01,394 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3d21c225, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T09:00:01,394 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request be1eb2620e0e,44373,-1 for getting cluster id 2024-11-20T09:00:01,394 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T09:00:01,395 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'caf50035-b19e-4cfd-90ea-dca559866882' 2024-11-20T09:00:01,395 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T09:00:01,395 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "caf50035-b19e-4cfd-90ea-dca559866882" 2024-11-20T09:00:01,395 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@69186cb8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T09:00:01,395 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [be1eb2620e0e,44373,-1] 2024-11-20T09:00:01,395 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T09:00:01,395 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T09:00:01,396 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32768, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T09:00:01,397 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46c9635f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T09:00:01,397 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a92e196, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T09:00:01,397 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request be1eb2620e0e,44373,-1 for getting cluster id 2024-11-20T09:00:01,397 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T09:00:01,398 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'caf50035-b19e-4cfd-90ea-dca559866882' 2024-11-20T09:00:01,398 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T09:00:01,398 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "caf50035-b19e-4cfd-90ea-dca559866882" 2024-11-20T09:00:01,398 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10be37a3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T09:00:01,398 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [be1eb2620e0e,44373,-1] 2024-11-20T09:00:01,398 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T09:00:01,399 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T09:00:01,399 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32790, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T09:00:01,400 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2966ad20, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T09:00:01,400 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3806f988, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T09:00:01,400 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request be1eb2620e0e,44373,-1 for getting cluster id 2024-11-20T09:00:01,400 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T09:00:01,401 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'caf50035-b19e-4cfd-90ea-dca559866882' 2024-11-20T09:00:01,401 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T09:00:01,401 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "caf50035-b19e-4cfd-90ea-dca559866882" 2024-11-20T09:00:01,401 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@550d85d5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T09:00:01,401 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [be1eb2620e0e,44373,-1] 2024-11-20T09:00:01,402 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T09:00:01,402 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T09:00:01,402 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32814, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T09:00:01,403 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f2e36da, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T09:00:01,403 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@494eb83f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T09:00:01,403 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request be1eb2620e0e,44373,-1 for getting cluster id 2024-11-20T09:00:01,404 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T09:00:01,404 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'caf50035-b19e-4cfd-90ea-dca559866882' 2024-11-20T09:00:01,404 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T09:00:01,404 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "caf50035-b19e-4cfd-90ea-dca559866882" 2024-11-20T09:00:01,405 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@231e1e41, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T09:00:01,405 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [be1eb2620e0e,44373,-1] 2024-11-20T09:00:01,405 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T09:00:01,405 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T09:00:01,405 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32844, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T09:00:01,406 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b8288ab, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T09:00:01,407 DEBUG [Thread-717 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T09:00:01,407 DEBUG [Thread-704 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T09:00:01,407 DEBUG [Thread-721 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T09:00:01,407 DEBUG [Thread-706 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T09:00:01,407 DEBUG [Thread-715 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T09:00:01,407 DEBUG [Thread-719 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T09:00:01,408 DEBUG [Thread-712 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T09:00:01,408 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=be1eb2620e0e,34223,1732093159960, seqNum=-1] 2024-11-20T09:00:01,408 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=be1eb2620e0e,34223,1732093159960, seqNum=-1] 2024-11-20T09:00:01,408 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=be1eb2620e0e,34223,1732093159960, seqNum=-1] 2024-11-20T09:00:01,408 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=be1eb2620e0e,34223,1732093159960, seqNum=-1] 2024-11-20T09:00:01,408 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T09:00:01,408 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T09:00:01,408 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T09:00:01,409 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T09:00:01,409 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=be1eb2620e0e,34223,1732093159960, seqNum=-1] 2024-11-20T09:00:01,409 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=be1eb2620e0e,34223,1732093159960, seqNum=-1] 2024-11-20T09:00:01,409 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T09:00:01,409 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T09:00:01,410 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=be1eb2620e0e,34223,1732093159960, seqNum=-1] 2024-11-20T09:00:01,410 DEBUG [Thread-710 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T09:00:01,410 DEBUG [Thread-717 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T09:00:01,410 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48394, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T09:00:01,410 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48388, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T09:00:01,411 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48404, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T09:00:01,411 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48420, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T09:00:01,411 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48406, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T09:00:01,411 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48424, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T09:00:01,411 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=be1eb2620e0e,34223,1732093159960, seqNum=-1] 2024-11-20T09:00:01,411 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T09:00:01,411 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48434, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T09:00:01,411 DEBUG [Thread-708 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T09:00:01,411 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T09:00:01,412 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48450, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T09:00:01,412 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=be1eb2620e0e,34223,1732093159960, seqNum=-1] 2024-11-20T09:00:01,413 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T09:00:01,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] procedure2.ProcedureExecutor(1139): Stored pid=36, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees 2024-11-20T09:00:01,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-20T09:00:01,413 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48466, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T09:00:01,414 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=36, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T09:00:01,414 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5] 2024-11-20T09:00:01,414 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5] 2024-11-20T09:00:01,415 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5] 2024-11-20T09:00:01,415 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5] 2024-11-20T09:00:01,416 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=36, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T09:00:01,417 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=36, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T09:00:01,417 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5] 2024-11-20T09:00:01,417 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5] 2024-11-20T09:00:01,418 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5] 2024-11-20T09:00:01,418 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5] 2024-11-20T09:00:01,418 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5] 2024-11-20T09:00:01,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(8855): Flush requested on f0e725d6c797c04eb86dfc3be87ab3ba 2024-11-20T09:00:01,433 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T09:00:01,433 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f0e725d6c797c04eb86dfc3be87ab3ba 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-20T09:00:01,485 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112073d03e8b9ba34ee39175adb74db7db5f_f0e725d6c797c04eb86dfc3be87ab3ba is 50, key is test_row_0/A:col10/1732093201428/Put/seqid=0 2024-11-20T09:00:01,492 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073741968_1144 (size=14594) 2024-11-20T09:00:01,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-20T09:00:01,543 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:01,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48466 deadline: 1732093261520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:01,544 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:01,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48394 deadline: 1732093261520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:01,545 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:01,545 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:01,545 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:01,545 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:01,545 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:01,545 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:01,546 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:01,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48420 deadline: 1732093261542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:01,546 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:01,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48450 deadline: 1732093261544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:01,547 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:01,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48388 deadline: 1732093261544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:01,548 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:01,548 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:01,548 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:01,548 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:01,548 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:01,548 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:01,548 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:01,548 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:01,548 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:01,571 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-20T09:00:01,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. 2024-11-20T09:00:01,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. as already flushing 2024-11-20T09:00:01,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. 2024-11-20T09:00:01,572 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => f0e725d6c797c04eb86dfc3be87ab3ba, NAME => 'TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:01,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => f0e725d6c797c04eb86dfc3be87ab3ba, NAME => 'TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:01,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0e725d6c797c04eb86dfc3be87ab3ba, NAME => 'TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0e725d6c797c04eb86dfc3be87ab3ba, NAME => 'TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:01,647 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:01,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48466 deadline: 1732093261647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:01,648 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:01,648 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:01,648 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:01,658 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:01,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48420 deadline: 1732093261657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:01,659 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:01,659 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:01,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48450 deadline: 1732093261658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:01,659 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:01,659 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:01,660 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:01,660 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:01,660 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:01,660 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:01,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48388 deadline: 1732093261658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:01,660 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:01,661 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:01,660 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:01,661 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:01,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48394 deadline: 1732093261658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:01,661 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:01,661 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:01,661 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:01,726 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-20T09:00:01,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. 2024-11-20T09:00:01,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. as already flushing 2024-11-20T09:00:01,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. 2024-11-20T09:00:01,727 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => f0e725d6c797c04eb86dfc3be87ab3ba, NAME => 'TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:01,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => f0e725d6c797c04eb86dfc3be87ab3ba, NAME => 'TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:01,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0e725d6c797c04eb86dfc3be87ab3ba, NAME => 'TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0e725d6c797c04eb86dfc3be87ab3ba, NAME => 'TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:01,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-20T09:00:01,858 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:01,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48466 deadline: 1732093261857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:01,859 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:01,859 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:01,859 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:01,869 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:01,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48420 deadline: 1732093261867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:01,869 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:01,870 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:01,870 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:01,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48450 deadline: 1732093261867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:01,870 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:01,870 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:01,870 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:01,870 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:01,870 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:01,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48394 deadline: 1732093261867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:01,871 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:01,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48388 deadline: 1732093261868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:01,871 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:01,871 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:01,872 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:01,872 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:01,872 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:01,872 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:01,881 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-20T09:00:01,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. 2024-11-20T09:00:01,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. as already flushing 2024-11-20T09:00:01,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. 2024-11-20T09:00:01,884 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => f0e725d6c797c04eb86dfc3be87ab3ba, NAME => 'TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:01,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => f0e725d6c797c04eb86dfc3be87ab3ba, NAME => 'TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:01,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0e725d6c797c04eb86dfc3be87ab3ba, NAME => 'TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0e725d6c797c04eb86dfc3be87ab3ba, NAME => 'TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:01,894 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:01,900 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112073d03e8b9ba34ee39175adb74db7db5f_f0e725d6c797c04eb86dfc3be87ab3ba to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112073d03e8b9ba34ee39175adb74db7db5f_f0e725d6c797c04eb86dfc3be87ab3ba 2024-11-20T09:00:01,902 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/A/66893ac4f3af4f0eb7e3d13bd68f287d, store: [table=TestAcidGuarantees family=A region=f0e725d6c797c04eb86dfc3be87ab3ba] 2024-11-20T09:00:01,913 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/A/66893ac4f3af4f0eb7e3d13bd68f287d is 175, key is test_row_0/A:col10/1732093201428/Put/seqid=0 2024-11-20T09:00:01,941 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073741969_1145 (size=39549) 2024-11-20T09:00:01,945 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=16, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/A/66893ac4f3af4f0eb7e3d13bd68f287d 2024-11-20T09:00:01,988 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/B/4832cd642c74480c9642917a950df0d3 is 50, key is test_row_0/B:col10/1732093201428/Put/seqid=0 2024-11-20T09:00:02,018 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073741970_1146 (size=12001) 2024-11-20T09:00:02,021 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/B/4832cd642c74480c9642917a950df0d3 2024-11-20T09:00:02,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-20T09:00:02,038 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-20T09:00:02,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. 2024-11-20T09:00:02,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. as already flushing 2024-11-20T09:00:02,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. 2024-11-20T09:00:02,039 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => f0e725d6c797c04eb86dfc3be87ab3ba, NAME => 'TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:02,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => f0e725d6c797c04eb86dfc3be87ab3ba, NAME => 'TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:02,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0e725d6c797c04eb86dfc3be87ab3ba, NAME => 'TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0e725d6c797c04eb86dfc3be87ab3ba, NAME => 'TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:02,086 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/C/3e6de594cc9e4503a7378c8133a5ef86 is 50, key is test_row_0/C:col10/1732093201428/Put/seqid=0 2024-11-20T09:00:02,125 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073741971_1147 (size=12001) 2024-11-20T09:00:02,127 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/C/3e6de594cc9e4503a7378c8133a5ef86 2024-11-20T09:00:02,137 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/A/66893ac4f3af4f0eb7e3d13bd68f287d as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/A/66893ac4f3af4f0eb7e3d13bd68f287d 2024-11-20T09:00:02,145 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/A/66893ac4f3af4f0eb7e3d13bd68f287d, entries=200, sequenceid=16, filesize=38.6 K 2024-11-20T09:00:02,146 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/B/4832cd642c74480c9642917a950df0d3 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/B/4832cd642c74480c9642917a950df0d3 2024-11-20T09:00:02,175 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:02,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48466 deadline: 1732093262167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:02,176 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:02,176 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:02,176 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/B/4832cd642c74480c9642917a950df0d3, entries=150, sequenceid=16, filesize=11.7 K 2024-11-20T09:00:02,176 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:02,179 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/C/3e6de594cc9e4503a7378c8133a5ef86 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/C/3e6de594cc9e4503a7378c8133a5ef86 2024-11-20T09:00:02,186 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/C/3e6de594cc9e4503a7378c8133a5ef86, entries=150, sequenceid=16, filesize=11.7 K 2024-11-20T09:00:02,186 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:02,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48388 deadline: 1732093262178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:02,189 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:02,189 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:02,189 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:02,190 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=147.60 KB/151140 for f0e725d6c797c04eb86dfc3be87ab3ba in 756ms, sequenceid=16, compaction requested=false 2024-11-20T09:00:02,190 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-20T09:00:02,191 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f0e725d6c797c04eb86dfc3be87ab3ba: 2024-11-20T09:00:02,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(8855): Flush requested on f0e725d6c797c04eb86dfc3be87ab3ba 2024-11-20T09:00:02,192 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T09:00:02,192 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f0e725d6c797c04eb86dfc3be87ab3ba 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-20T09:00:02,193 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-20T09:00:02,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. 2024-11-20T09:00:02,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. as already flushing 2024-11-20T09:00:02,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. 2024-11-20T09:00:02,193 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => f0e725d6c797c04eb86dfc3be87ab3ba, NAME => 'TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:02,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => f0e725d6c797c04eb86dfc3be87ab3ba, NAME => 'TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:02,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0e725d6c797c04eb86dfc3be87ab3ba, NAME => 'TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0e725d6c797c04eb86dfc3be87ab3ba, NAME => 'TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:02,225 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411200f3103f23afa478ab84384d04d1bbf7c_f0e725d6c797c04eb86dfc3be87ab3ba is 50, key is test_row_0/A:col10/1732093201541/Put/seqid=0 2024-11-20T09:00:02,229 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:02,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48420 deadline: 1732093262225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:02,230 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:02,230 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:02,230 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:02,235 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:02,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48394 deadline: 1732093262228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:02,236 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:02,236 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:02,236 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:02,237 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:02,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48450 deadline: 1732093262236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:02,238 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:02,238 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:02,238 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:02,262 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073741972_1148 (size=12154) 2024-11-20T09:00:02,264 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:02,271 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411200f3103f23afa478ab84384d04d1bbf7c_f0e725d6c797c04eb86dfc3be87ab3ba to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411200f3103f23afa478ab84384d04d1bbf7c_f0e725d6c797c04eb86dfc3be87ab3ba 2024-11-20T09:00:02,272 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/A/d7e57c57363942fc809ffc2f05441e30, store: [table=TestAcidGuarantees family=A region=f0e725d6c797c04eb86dfc3be87ab3ba] 2024-11-20T09:00:02,273 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/A/d7e57c57363942fc809ffc2f05441e30 is 175, key is test_row_0/A:col10/1732093201541/Put/seqid=0 2024-11-20T09:00:02,295 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073741973_1149 (size=30955) 2024-11-20T09:00:02,345 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:02,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48420 deadline: 1732093262337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:02,347 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:02,347 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:02,347 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:02,348 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-20T09:00:02,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. 2024-11-20T09:00:02,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. as already flushing 2024-11-20T09:00:02,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. 2024-11-20T09:00:02,349 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => f0e725d6c797c04eb86dfc3be87ab3ba, NAME => 'TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:02,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => f0e725d6c797c04eb86dfc3be87ab3ba, NAME => 'TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:02,348 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:02,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48394 deadline: 1732093262347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:02,349 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:02,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48450 deadline: 1732093262347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:02,350 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:02,350 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:02,350 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:02,350 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:02,350 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:02,350 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:02,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0e725d6c797c04eb86dfc3be87ab3ba, NAME => 'TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0e725d6c797c04eb86dfc3be87ab3ba, NAME => 'TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:02,503 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-20T09:00:02,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. 2024-11-20T09:00:02,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. as already flushing 2024-11-20T09:00:02,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. 2024-11-20T09:00:02,504 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => f0e725d6c797c04eb86dfc3be87ab3ba, NAME => 'TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:02,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => f0e725d6c797c04eb86dfc3be87ab3ba, NAME => 'TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:02,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0e725d6c797c04eb86dfc3be87ab3ba, NAME => 'TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0e725d6c797c04eb86dfc3be87ab3ba, NAME => 'TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:02,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-20T09:00:02,557 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:02,558 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:02,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48420 deadline: 1732093262557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:02,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48394 deadline: 1732093262557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:02,558 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:02,559 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:02,559 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:02,559 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:02,559 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:02,559 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:02,559 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:02,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48450 deadline: 1732093262557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:02,560 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:02,560 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:02,560 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:02,659 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-20T09:00:02,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. 2024-11-20T09:00:02,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. as already flushing 2024-11-20T09:00:02,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. 2024-11-20T09:00:02,660 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => f0e725d6c797c04eb86dfc3be87ab3ba, NAME => 'TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:02,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => f0e725d6c797c04eb86dfc3be87ab3ba, NAME => 'TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:02,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0e725d6c797c04eb86dfc3be87ab3ba, NAME => 'TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0e725d6c797c04eb86dfc3be87ab3ba, NAME => 'TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:02,688 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:02,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48466 deadline: 1732093262687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:02,691 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:02,691 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:02,691 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:02,696 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=43, memsize=53.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/A/d7e57c57363942fc809ffc2f05441e30 2024-11-20T09:00:02,699 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:02,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48388 deadline: 1732093262697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:02,700 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:02,700 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:02,700 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:02,718 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/B/d65db5654ca24639a731ed343d9e4dfe is 50, key is test_row_0/B:col10/1732093201541/Put/seqid=0 2024-11-20T09:00:02,751 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073741974_1150 (size=12001) 2024-11-20T09:00:02,753 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=43 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/B/d65db5654ca24639a731ed343d9e4dfe 2024-11-20T09:00:02,773 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/C/7bd85b04e36c47d89ce8b634e3336d35 is 50, key is test_row_0/C:col10/1732093201541/Put/seqid=0 2024-11-20T09:00:02,789 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073741975_1151 (size=12001) 2024-11-20T09:00:02,793 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=43 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/C/7bd85b04e36c47d89ce8b634e3336d35 2024-11-20T09:00:02,800 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/A/d7e57c57363942fc809ffc2f05441e30 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/A/d7e57c57363942fc809ffc2f05441e30 2024-11-20T09:00:02,809 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/A/d7e57c57363942fc809ffc2f05441e30, entries=150, sequenceid=43, filesize=30.2 K 2024-11-20T09:00:02,810 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/B/d65db5654ca24639a731ed343d9e4dfe as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/B/d65db5654ca24639a731ed343d9e4dfe 2024-11-20T09:00:02,814 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-20T09:00:02,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. 2024-11-20T09:00:02,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. as already flushing 2024-11-20T09:00:02,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. 2024-11-20T09:00:02,815 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => f0e725d6c797c04eb86dfc3be87ab3ba, NAME => 'TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:02,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => f0e725d6c797c04eb86dfc3be87ab3ba, NAME => 'TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:02,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0e725d6c797c04eb86dfc3be87ab3ba, NAME => 'TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0e725d6c797c04eb86dfc3be87ab3ba, NAME => 'TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:02,823 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/B/d65db5654ca24639a731ed343d9e4dfe, entries=150, sequenceid=43, filesize=11.7 K 2024-11-20T09:00:02,833 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/C/7bd85b04e36c47d89ce8b634e3336d35 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/C/7bd85b04e36c47d89ce8b634e3336d35 2024-11-20T09:00:02,857 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/C/7bd85b04e36c47d89ce8b634e3336d35, entries=150, sequenceid=43, filesize=11.7 K 2024-11-20T09:00:02,858 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=46.96 KB/48090 for f0e725d6c797c04eb86dfc3be87ab3ba in 666ms, sequenceid=43, compaction requested=false 2024-11-20T09:00:02,858 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f0e725d6c797c04eb86dfc3be87ab3ba: 2024-11-20T09:00:02,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(8855): Flush requested on f0e725d6c797c04eb86dfc3be87ab3ba 2024-11-20T09:00:02,873 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T09:00:02,873 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f0e725d6c797c04eb86dfc3be87ab3ba 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-20T09:00:02,915 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120f11506a113fa40849aaadb26f853ed6f_f0e725d6c797c04eb86dfc3be87ab3ba is 50, key is test_row_0/A:col10/1732093202873/Put/seqid=0 2024-11-20T09:00:02,943 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073741976_1152 (size=14594) 2024-11-20T09:00:02,945 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:02,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48450 deadline: 1732093262936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:02,946 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:02,947 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:02,947 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:02,954 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:02,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48420 deadline: 1732093262946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:02,955 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:02,955 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:02,955 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:02,956 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:02,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48394 deadline: 1732093262947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:02,957 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:02,957 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:02,957 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:02,970 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-20T09:00:02,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. 2024-11-20T09:00:02,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. as already flushing 2024-11-20T09:00:02,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. 2024-11-20T09:00:02,971 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => f0e725d6c797c04eb86dfc3be87ab3ba, NAME => 'TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:02,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => f0e725d6c797c04eb86dfc3be87ab3ba, NAME => 'TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:02,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0e725d6c797c04eb86dfc3be87ab3ba, NAME => 'TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0e725d6c797c04eb86dfc3be87ab3ba, NAME => 'TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:03,058 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:03,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48450 deadline: 1732093263057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:03,059 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:03,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48420 deadline: 1732093263057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:03,059 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:03,059 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:03,059 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:03,059 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:03,059 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:03,059 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:03,075 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:03,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48394 deadline: 1732093263070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:03,076 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:03,076 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:03,077 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:03,126 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-20T09:00:03,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. 2024-11-20T09:00:03,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. as already flushing 2024-11-20T09:00:03,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. 2024-11-20T09:00:03,127 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => f0e725d6c797c04eb86dfc3be87ab3ba, NAME => 'TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:03,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => f0e725d6c797c04eb86dfc3be87ab3ba, NAME => 'TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:03,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0e725d6c797c04eb86dfc3be87ab3ba, NAME => 'TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0e725d6c797c04eb86dfc3be87ab3ba, NAME => 'TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:03,143 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-20T09:00:03,273 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:03,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48450 deadline: 1732093263268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:03,273 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:03,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48420 deadline: 1732093263268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:03,274 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:03,274 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:03,274 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:03,274 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:03,274 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:03,274 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:03,281 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-20T09:00:03,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. 2024-11-20T09:00:03,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. as already flushing 2024-11-20T09:00:03,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. 2024-11-20T09:00:03,282 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => f0e725d6c797c04eb86dfc3be87ab3ba, NAME => 'TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:03,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => f0e725d6c797c04eb86dfc3be87ab3ba, NAME => 'TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:03,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0e725d6c797c04eb86dfc3be87ab3ba, NAME => 'TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0e725d6c797c04eb86dfc3be87ab3ba, NAME => 'TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:03,291 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:03,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48394 deadline: 1732093263287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:03,294 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:03,294 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:03,294 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:03,346 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:03,351 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120f11506a113fa40849aaadb26f853ed6f_f0e725d6c797c04eb86dfc3be87ab3ba to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120f11506a113fa40849aaadb26f853ed6f_f0e725d6c797c04eb86dfc3be87ab3ba 2024-11-20T09:00:03,353 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/A/e90cf813d9b24f4f85d4f03efc79f52e, store: [table=TestAcidGuarantees family=A region=f0e725d6c797c04eb86dfc3be87ab3ba] 2024-11-20T09:00:03,354 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/A/e90cf813d9b24f4f85d4f03efc79f52e is 175, key is test_row_0/A:col10/1732093202873/Put/seqid=0 2024-11-20T09:00:03,369 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073741977_1153 (size=39549) 2024-11-20T09:00:03,371 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=56, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/A/e90cf813d9b24f4f85d4f03efc79f52e 2024-11-20T09:00:03,384 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/B/4eab81e2d5d34df589f6de8c4086c355 is 50, key is test_row_0/B:col10/1732093202873/Put/seqid=0 2024-11-20T09:00:03,397 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073741978_1154 (size=12001) 2024-11-20T09:00:03,399 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/B/4eab81e2d5d34df589f6de8c4086c355 2024-11-20T09:00:03,412 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/C/2b6dcdb6782543a0aaf59181e6b850ac is 50, key is test_row_0/C:col10/1732093202873/Put/seqid=0 2024-11-20T09:00:03,432 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073741979_1155 (size=12001) 2024-11-20T09:00:03,433 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/C/2b6dcdb6782543a0aaf59181e6b850ac 2024-11-20T09:00:03,438 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-20T09:00:03,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. 2024-11-20T09:00:03,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. as already flushing 2024-11-20T09:00:03,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. 2024-11-20T09:00:03,439 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => f0e725d6c797c04eb86dfc3be87ab3ba, NAME => 'TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:03,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => f0e725d6c797c04eb86dfc3be87ab3ba, NAME => 'TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:03,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0e725d6c797c04eb86dfc3be87ab3ba, NAME => 'TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0e725d6c797c04eb86dfc3be87ab3ba, NAME => 'TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:03,447 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/A/e90cf813d9b24f4f85d4f03efc79f52e as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/A/e90cf813d9b24f4f85d4f03efc79f52e 2024-11-20T09:00:03,460 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/A/e90cf813d9b24f4f85d4f03efc79f52e, entries=200, sequenceid=56, filesize=38.6 K 2024-11-20T09:00:03,461 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/B/4eab81e2d5d34df589f6de8c4086c355 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/B/4eab81e2d5d34df589f6de8c4086c355 2024-11-20T09:00:03,464 INFO [master/be1eb2620e0e:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-20T09:00:03,464 INFO [master/be1eb2620e0e:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-20T09:00:03,466 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/B/4eab81e2d5d34df589f6de8c4086c355, entries=150, sequenceid=56, filesize=11.7 K 2024-11-20T09:00:03,467 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/C/2b6dcdb6782543a0aaf59181e6b850ac as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/C/2b6dcdb6782543a0aaf59181e6b850ac 2024-11-20T09:00:03,475 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/C/2b6dcdb6782543a0aaf59181e6b850ac, entries=150, sequenceid=56, filesize=11.7 K 2024-11-20T09:00:03,478 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=140.89 KB/144270 for f0e725d6c797c04eb86dfc3be87ab3ba in 605ms, sequenceid=56, compaction requested=true 2024-11-20T09:00:03,478 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f0e725d6c797c04eb86dfc3be87ab3ba: 2024-11-20T09:00:03,479 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T09:00:03,479 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f0e725d6c797c04eb86dfc3be87ab3ba:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T09:00:03,479 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T09:00:03,479 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T09:00:03,479 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f0e725d6c797c04eb86dfc3be87ab3ba:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T09:00:03,479 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T09:00:03,479 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f0e725d6c797c04eb86dfc3be87ab3ba:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T09:00:03,479 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T09:00:03,483 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T09:00:03,483 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1541): f0e725d6c797c04eb86dfc3be87ab3ba/B is initiating minor compaction (all files) 2024-11-20T09:00:03,483 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f0e725d6c797c04eb86dfc3be87ab3ba/B in TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. 2024-11-20T09:00:03,483 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/B/4832cd642c74480c9642917a950df0d3, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/B/d65db5654ca24639a731ed343d9e4dfe, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/B/4eab81e2d5d34df589f6de8c4086c355] into tmpdir=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp, totalSize=35.2 K 2024-11-20T09:00:03,484 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110053 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T09:00:03,484 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HStore(1541): f0e725d6c797c04eb86dfc3be87ab3ba/A is initiating minor compaction (all files) 2024-11-20T09:00:03,484 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f0e725d6c797c04eb86dfc3be87ab3ba/A in TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. 2024-11-20T09:00:03,484 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/A/66893ac4f3af4f0eb7e3d13bd68f287d, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/A/d7e57c57363942fc809ffc2f05441e30, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/A/e90cf813d9b24f4f85d4f03efc79f52e] into tmpdir=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp, totalSize=107.5 K 2024-11-20T09:00:03,484 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting 4832cd642c74480c9642917a950df0d3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1732093201417 2024-11-20T09:00:03,484 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. 2024-11-20T09:00:03,484 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. files: [hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/A/66893ac4f3af4f0eb7e3d13bd68f287d, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/A/d7e57c57363942fc809ffc2f05441e30, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/A/e90cf813d9b24f4f85d4f03efc79f52e] 2024-11-20T09:00:03,485 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting d65db5654ca24639a731ed343d9e4dfe, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=43, earliestPutTs=1732093201541 2024-11-20T09:00:03,485 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.Compactor(225): Compacting 66893ac4f3af4f0eb7e3d13bd68f287d, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1732093201417 2024-11-20T09:00:03,485 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting 4eab81e2d5d34df589f6de8c4086c355, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1732093202212 2024-11-20T09:00:03,486 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.Compactor(225): Compacting d7e57c57363942fc809ffc2f05441e30, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=43, earliestPutTs=1732093201541 2024-11-20T09:00:03,487 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.Compactor(225): Compacting e90cf813d9b24f4f85d4f03efc79f52e, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1732093202212 2024-11-20T09:00:03,506 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f0e725d6c797c04eb86dfc3be87ab3ba#B#compaction#137 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T09:00:03,506 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/B/2dab945a9c664ebb88f30ee634af33a5 is 50, key is test_row_0/B:col10/1732093202873/Put/seqid=0 2024-11-20T09:00:03,513 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=f0e725d6c797c04eb86dfc3be87ab3ba] 2024-11-20T09:00:03,521 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241120c27cc3098d024904aa34ce5db328ad1a_f0e725d6c797c04eb86dfc3be87ab3ba store=[table=TestAcidGuarantees family=A region=f0e725d6c797c04eb86dfc3be87ab3ba] 2024-11-20T09:00:03,526 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241120c27cc3098d024904aa34ce5db328ad1a_f0e725d6c797c04eb86dfc3be87ab3ba, store=[table=TestAcidGuarantees family=A region=f0e725d6c797c04eb86dfc3be87ab3ba] 2024-11-20T09:00:03,527 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120c27cc3098d024904aa34ce5db328ad1a_f0e725d6c797c04eb86dfc3be87ab3ba because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=f0e725d6c797c04eb86dfc3be87ab3ba] 2024-11-20T09:00:03,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-20T09:00:03,559 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073741980_1156 (size=12104) 2024-11-20T09:00:03,570 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073741981_1157 (size=4469) 2024-11-20T09:00:03,572 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f0e725d6c797c04eb86dfc3be87ab3ba#A#compaction#138 average throughput is 0.41 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T09:00:03,573 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/A/2f744b70760844c9ac03430f5eeeedc4 is 175, key is test_row_0/A:col10/1732093202873/Put/seqid=0 2024-11-20T09:00:03,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(8855): Flush requested on f0e725d6c797c04eb86dfc3be87ab3ba 2024-11-20T09:00:03,581 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T09:00:03,581 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f0e725d6c797c04eb86dfc3be87ab3ba 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-20T09:00:03,593 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-20T09:00:03,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. 2024-11-20T09:00:03,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. as already flushing 2024-11-20T09:00:03,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. 2024-11-20T09:00:03,593 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => f0e725d6c797c04eb86dfc3be87ab3ba, NAME => 'TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:03,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => f0e725d6c797c04eb86dfc3be87ab3ba, NAME => 'TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:03,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0e725d6c797c04eb86dfc3be87ab3ba, NAME => 'TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0e725d6c797c04eb86dfc3be87ab3ba, NAME => 'TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:03,606 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:03,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48450 deadline: 1732093263603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:03,607 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:03,607 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:03,607 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:03,607 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:03,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48394 deadline: 1732093263604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:03,608 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:03,608 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:03,608 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:03,613 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073741982_1158 (size=31058) 2024-11-20T09:00:03,614 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:03,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48420 deadline: 1732093263606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:03,615 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:03,615 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:03,615 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:03,623 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120799c0e309fac4ab3954500cbc6b5b52e_f0e725d6c797c04eb86dfc3be87ab3ba is 50, key is test_row_0/A:col10/1732093203581/Put/seqid=0 2024-11-20T09:00:03,625 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/A/2f744b70760844c9ac03430f5eeeedc4 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/A/2f744b70760844c9ac03430f5eeeedc4 2024-11-20T09:00:03,632 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f0e725d6c797c04eb86dfc3be87ab3ba/A of f0e725d6c797c04eb86dfc3be87ab3ba into 2f744b70760844c9ac03430f5eeeedc4(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T09:00:03,632 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f0e725d6c797c04eb86dfc3be87ab3ba: 2024-11-20T09:00:03,632 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., storeName=f0e725d6c797c04eb86dfc3be87ab3ba/A, priority=13, startTime=1732093203478; duration=0sec 2024-11-20T09:00:03,633 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T09:00:03,633 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f0e725d6c797c04eb86dfc3be87ab3ba:A 2024-11-20T09:00:03,633 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T09:00:03,634 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T09:00:03,634 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HStore(1541): f0e725d6c797c04eb86dfc3be87ab3ba/C is initiating minor compaction (all files) 2024-11-20T09:00:03,634 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f0e725d6c797c04eb86dfc3be87ab3ba/C in TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. 2024-11-20T09:00:03,635 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/C/3e6de594cc9e4503a7378c8133a5ef86, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/C/7bd85b04e36c47d89ce8b634e3336d35, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/C/2b6dcdb6782543a0aaf59181e6b850ac] into tmpdir=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp, totalSize=35.2 K 2024-11-20T09:00:03,635 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3e6de594cc9e4503a7378c8133a5ef86, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1732093201417 2024-11-20T09:00:03,636 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7bd85b04e36c47d89ce8b634e3336d35, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=43, earliestPutTs=1732093201541 2024-11-20T09:00:03,636 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2b6dcdb6782543a0aaf59181e6b850ac, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1732093202212 2024-11-20T09:00:03,655 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f0e725d6c797c04eb86dfc3be87ab3ba#C#compaction#140 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T09:00:03,656 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/C/3126051b8ac2415593113cc36e85695f is 50, key is test_row_0/C:col10/1732093202873/Put/seqid=0 2024-11-20T09:00:03,662 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073741983_1159 (size=17034) 2024-11-20T09:00:03,663 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:03,669 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120799c0e309fac4ab3954500cbc6b5b52e_f0e725d6c797c04eb86dfc3be87ab3ba to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120799c0e309fac4ab3954500cbc6b5b52e_f0e725d6c797c04eb86dfc3be87ab3ba 2024-11-20T09:00:03,670 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/A/6a279628dca84f46b8b6eddb02457932, store: [table=TestAcidGuarantees family=A region=f0e725d6c797c04eb86dfc3be87ab3ba] 2024-11-20T09:00:03,671 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/A/6a279628dca84f46b8b6eddb02457932 is 175, key is test_row_0/A:col10/1732093203581/Put/seqid=0 2024-11-20T09:00:03,696 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073741984_1160 (size=12104) 2024-11-20T09:00:03,698 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:03,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48466 deadline: 1732093263697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:03,701 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to be1eb2620e0e,34223,1732093159960 for 'test_row_1' in f0e725d6c797c04eb86dfc3be87ab3ba of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2181 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-20T09:00:03,702 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:03,702 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:03,702 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:03,703 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/C/3126051b8ac2415593113cc36e85695f as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/C/3126051b8ac2415593113cc36e85695f 2024-11-20T09:00:03,711 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f0e725d6c797c04eb86dfc3be87ab3ba/C of f0e725d6c797c04eb86dfc3be87ab3ba into 3126051b8ac2415593113cc36e85695f(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T09:00:03,711 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f0e725d6c797c04eb86dfc3be87ab3ba: 2024-11-20T09:00:03,711 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., storeName=f0e725d6c797c04eb86dfc3be87ab3ba/C, priority=13, startTime=1732093203479; duration=0sec 2024-11-20T09:00:03,712 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T09:00:03,712 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f0e725d6c797c04eb86dfc3be87ab3ba:C 2024-11-20T09:00:03,712 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:03,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48388 deadline: 1732093263707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:03,713 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to be1eb2620e0e,34223,1732093159960 for 'test_row_0' in f0e725d6c797c04eb86dfc3be87ab3ba of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2169 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-20T09:00:03,714 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:03,714 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:03,714 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:03,718 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:03,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48450 deadline: 1732093263717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:03,719 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:03,719 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:03,719 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:03,719 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:03,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48394 deadline: 1732093263718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:03,719 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:03,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48420 deadline: 1732093263718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:03,720 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:03,720 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:03,720 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:03,720 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:03,720 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:03,720 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:03,726 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073741985_1161 (size=48139) 2024-11-20T09:00:03,750 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-20T09:00:03,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. 2024-11-20T09:00:03,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. as already flushing 2024-11-20T09:00:03,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. 2024-11-20T09:00:03,751 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => f0e725d6c797c04eb86dfc3be87ab3ba, NAME => 'TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:03,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => f0e725d6c797c04eb86dfc3be87ab3ba, NAME => 'TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:03,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0e725d6c797c04eb86dfc3be87ab3ba, NAME => 'TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0e725d6c797c04eb86dfc3be87ab3ba, NAME => 'TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:03,905 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-20T09:00:03,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. 2024-11-20T09:00:03,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. as already flushing 2024-11-20T09:00:03,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. 2024-11-20T09:00:03,906 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => f0e725d6c797c04eb86dfc3be87ab3ba, NAME => 'TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:03,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => f0e725d6c797c04eb86dfc3be87ab3ba, NAME => 'TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:03,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0e725d6c797c04eb86dfc3be87ab3ba, NAME => 'TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0e725d6c797c04eb86dfc3be87ab3ba, NAME => 'TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:03,929 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:03,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48450 deadline: 1732093263927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:03,930 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:03,929 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:03,930 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:03,930 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:03,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48394 deadline: 1732093263927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:03,930 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:03,930 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:03,930 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:03,930 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:03,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48420 deadline: 1732093263928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:03,931 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:03,931 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:03,931 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:03,968 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/B/2dab945a9c664ebb88f30ee634af33a5 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/B/2dab945a9c664ebb88f30ee634af33a5 2024-11-20T09:00:03,974 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f0e725d6c797c04eb86dfc3be87ab3ba/B of f0e725d6c797c04eb86dfc3be87ab3ba into 2dab945a9c664ebb88f30ee634af33a5(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T09:00:03,974 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f0e725d6c797c04eb86dfc3be87ab3ba: 2024-11-20T09:00:03,974 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., storeName=f0e725d6c797c04eb86dfc3be87ab3ba/B, priority=13, startTime=1732093203479; duration=0sec 2024-11-20T09:00:03,975 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T09:00:03,975 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f0e725d6c797c04eb86dfc3be87ab3ba:B 2024-11-20T09:00:04,059 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-20T09:00:04,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. 2024-11-20T09:00:04,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. as already flushing 2024-11-20T09:00:04,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. 2024-11-20T09:00:04,060 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => f0e725d6c797c04eb86dfc3be87ab3ba, NAME => 'TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:04,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => f0e725d6c797c04eb86dfc3be87ab3ba, NAME => 'TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:04,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0e725d6c797c04eb86dfc3be87ab3ba, NAME => 'TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0e725d6c797c04eb86dfc3be87ab3ba, NAME => 'TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:04,128 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=82, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/A/6a279628dca84f46b8b6eddb02457932 2024-11-20T09:00:04,146 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/B/4ee36236777344abba8cb58c74f4f910 is 50, key is test_row_0/B:col10/1732093203581/Put/seqid=0 2024-11-20T09:00:04,162 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073741986_1162 (size=12001) 2024-11-20T09:00:04,163 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/B/4ee36236777344abba8cb58c74f4f910 2024-11-20T09:00:04,183 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/C/0377e3b27ed74c2b8aa8a87dd9121431 is 50, key is test_row_0/C:col10/1732093203581/Put/seqid=0 2024-11-20T09:00:04,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073741987_1163 (size=12001) 2024-11-20T09:00:04,215 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-20T09:00:04,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. 2024-11-20T09:00:04,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. as already flushing 2024-11-20T09:00:04,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. 2024-11-20T09:00:04,216 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => f0e725d6c797c04eb86dfc3be87ab3ba, NAME => 'TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:04,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => f0e725d6c797c04eb86dfc3be87ab3ba, NAME => 'TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:04,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0e725d6c797c04eb86dfc3be87ab3ba, NAME => 'TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0e725d6c797c04eb86dfc3be87ab3ba, NAME => 'TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:04,238 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:04,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48394 deadline: 1732093264237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:04,239 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:04,239 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:04,239 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:04,239 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:04,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48450 deadline: 1732093264237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:04,239 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:04,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48420 deadline: 1732093264237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:04,240 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:04,240 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:04,240 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:04,240 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:04,240 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:04,240 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:04,368 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-20T09:00:04,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. 2024-11-20T09:00:04,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. as already flushing 2024-11-20T09:00:04,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. 2024-11-20T09:00:04,369 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => f0e725d6c797c04eb86dfc3be87ab3ba, NAME => 'TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:04,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => f0e725d6c797c04eb86dfc3be87ab3ba, NAME => 'TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:04,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0e725d6c797c04eb86dfc3be87ab3ba, NAME => 'TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0e725d6c797c04eb86dfc3be87ab3ba, NAME => 'TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:04,522 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-20T09:00:04,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. 2024-11-20T09:00:04,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. as already flushing 2024-11-20T09:00:04,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. 2024-11-20T09:00:04,522 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => f0e725d6c797c04eb86dfc3be87ab3ba, NAME => 'TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:04,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => f0e725d6c797c04eb86dfc3be87ab3ba, NAME => 'TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:04,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0e725d6c797c04eb86dfc3be87ab3ba, NAME => 'TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0e725d6c797c04eb86dfc3be87ab3ba, NAME => 'TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:04,595 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/C/0377e3b27ed74c2b8aa8a87dd9121431 2024-11-20T09:00:04,603 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/A/6a279628dca84f46b8b6eddb02457932 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/A/6a279628dca84f46b8b6eddb02457932 2024-11-20T09:00:04,610 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/A/6a279628dca84f46b8b6eddb02457932, entries=250, sequenceid=82, filesize=47.0 K 2024-11-20T09:00:04,613 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/B/4ee36236777344abba8cb58c74f4f910 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/B/4ee36236777344abba8cb58c74f4f910 2024-11-20T09:00:04,629 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/B/4ee36236777344abba8cb58c74f4f910, entries=150, sequenceid=82, filesize=11.7 K 2024-11-20T09:00:04,630 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/C/0377e3b27ed74c2b8aa8a87dd9121431 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/C/0377e3b27ed74c2b8aa8a87dd9121431 2024-11-20T09:00:04,638 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/C/0377e3b27ed74c2b8aa8a87dd9121431, entries=150, sequenceid=82, filesize=11.7 K 2024-11-20T09:00:04,641 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for f0e725d6c797c04eb86dfc3be87ab3ba in 1060ms, sequenceid=82, compaction requested=false 2024-11-20T09:00:04,641 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f0e725d6c797c04eb86dfc3be87ab3ba: 2024-11-20T09:00:04,677 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-20T09:00:04,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. 2024-11-20T09:00:04,677 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2902): Flushing f0e725d6c797c04eb86dfc3be87ab3ba 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-20T09:00:04,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112041f5b8801dc94e0f96fd85a022b02161_f0e725d6c797c04eb86dfc3be87ab3ba is 50, key is test_row_0/A:col10/1732093203602/Put/seqid=0 2024-11-20T09:00:04,717 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073741988_1164 (size=12154) 2024-11-20T09:00:04,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:04,733 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112041f5b8801dc94e0f96fd85a022b02161_f0e725d6c797c04eb86dfc3be87ab3ba to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112041f5b8801dc94e0f96fd85a022b02161_f0e725d6c797c04eb86dfc3be87ab3ba 2024-11-20T09:00:04,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/A/ef9a68d183ea47b5a6356aa1384ee3e8, store: [table=TestAcidGuarantees family=A region=f0e725d6c797c04eb86dfc3be87ab3ba] 2024-11-20T09:00:04,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/A/ef9a68d183ea47b5a6356aa1384ee3e8 is 175, key is test_row_0/A:col10/1732093203602/Put/seqid=0 2024-11-20T09:00:04,748 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073741989_1165 (size=30955) 2024-11-20T09:00:04,750 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. as already flushing 2024-11-20T09:00:04,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(8855): Flush requested on f0e725d6c797c04eb86dfc3be87ab3ba 2024-11-20T09:00:04,751 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=95, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/A/ef9a68d183ea47b5a6356aa1384ee3e8 2024-11-20T09:00:04,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/B/45abc1a01d1341d1a67cfe1881a3a929 is 50, key is test_row_0/B:col10/1732093203602/Put/seqid=0 2024-11-20T09:00:04,798 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073741990_1166 (size=12001) 2024-11-20T09:00:04,824 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/B/45abc1a01d1341d1a67cfe1881a3a929 2024-11-20T09:00:04,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/C/04fb40e09f9f4dfca658cfc049179d49 is 50, key is test_row_0/C:col10/1732093203602/Put/seqid=0 2024-11-20T09:00:04,849 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073741991_1167 (size=12001) 2024-11-20T09:00:04,850 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/C/04fb40e09f9f4dfca658cfc049179d49 2024-11-20T09:00:04,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/A/ef9a68d183ea47b5a6356aa1384ee3e8 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/A/ef9a68d183ea47b5a6356aa1384ee3e8 2024-11-20T09:00:04,859 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:04,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48420 deadline: 1732093264849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:04,860 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:04,860 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:04,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48394 deadline: 1732093264851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:04,860 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:04,860 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:04,861 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:04,861 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:04,861 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:04,864 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/A/ef9a68d183ea47b5a6356aa1384ee3e8, entries=150, sequenceid=95, filesize=30.2 K 2024-11-20T09:00:04,865 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:04,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48450 deadline: 1732093264860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:04,865 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:04,866 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:04,866 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:04,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/B/45abc1a01d1341d1a67cfe1881a3a929 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/B/45abc1a01d1341d1a67cfe1881a3a929 2024-11-20T09:00:04,874 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/B/45abc1a01d1341d1a67cfe1881a3a929, entries=150, sequenceid=95, filesize=11.7 K 2024-11-20T09:00:04,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/C/04fb40e09f9f4dfca658cfc049179d49 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/C/04fb40e09f9f4dfca658cfc049179d49 2024-11-20T09:00:04,883 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/C/04fb40e09f9f4dfca658cfc049179d49, entries=150, sequenceid=95, filesize=11.7 K 2024-11-20T09:00:04,888 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=154.31 KB/158010 for f0e725d6c797c04eb86dfc3be87ab3ba in 211ms, sequenceid=95, compaction requested=true 2024-11-20T09:00:04,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2603): Flush status journal for f0e725d6c797c04eb86dfc3be87ab3ba: 2024-11-20T09:00:04,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. 2024-11-20T09:00:04,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=37 2024-11-20T09:00:04,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.HMaster(4169): Remote procedure done, pid=37 2024-11-20T09:00:04,895 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=36 2024-11-20T09:00:04,895 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=36, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.4740 sec 2024-11-20T09:00:04,898 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees in 3.4850 sec 2024-11-20T09:00:04,971 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T09:00:04,971 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f0e725d6c797c04eb86dfc3be87ab3ba 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-20T09:00:04,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(8855): Flush requested on f0e725d6c797c04eb86dfc3be87ab3ba 2024-11-20T09:00:04,984 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:04,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48450 deadline: 1732093264982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:04,986 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:04,986 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:04,986 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:04,988 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:04,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48394 deadline: 1732093264983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:04,989 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:04,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48420 deadline: 1732093264985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:04,990 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:04,990 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:04,990 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:04,990 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:04,990 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:04,990 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:05,005 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411200baa6142cbf34b07a8f2cf11953340a3_f0e725d6c797c04eb86dfc3be87ab3ba is 50, key is test_row_0/A:col10/1732093204969/Put/seqid=0 2024-11-20T09:00:05,015 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073741992_1168 (size=14594) 2024-11-20T09:00:05,017 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,025 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411200baa6142cbf34b07a8f2cf11953340a3_f0e725d6c797c04eb86dfc3be87ab3ba to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411200baa6142cbf34b07a8f2cf11953340a3_f0e725d6c797c04eb86dfc3be87ab3ba 2024-11-20T09:00:05,026 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/A/c04675b5ee5b4bd3b555cb77a691f5d1, store: [table=TestAcidGuarantees family=A region=f0e725d6c797c04eb86dfc3be87ab3ba] 2024-11-20T09:00:05,027 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/A/c04675b5ee5b4bd3b555cb77a691f5d1 is 175, key is test_row_0/A:col10/1732093204969/Put/seqid=0 2024-11-20T09:00:05,038 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073741993_1169 (size=39549) 2024-11-20T09:00:05,040 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=122, memsize=53.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/A/c04675b5ee5b4bd3b555cb77a691f5d1 2024-11-20T09:00:05,069 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/B/b09224912527476ca924e92bc11b8a3f is 50, key is test_row_0/B:col10/1732093204969/Put/seqid=0 2024-11-20T09:00:05,079 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073741994_1170 (size=12001) 2024-11-20T09:00:05,080 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=122 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/B/b09224912527476ca924e92bc11b8a3f 2024-11-20T09:00:05,097 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/C/9d0779fadc434aaf868ac25f9fcb945a is 50, key is test_row_0/C:col10/1732093204969/Put/seqid=0 2024-11-20T09:00:05,102 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:05,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48450 deadline: 1732093265098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:05,103 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:05,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48420 deadline: 1732093265098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:05,103 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:05,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48394 deadline: 1732093265099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:05,104 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:05,104 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:05,104 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:05,104 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:05,104 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:05,104 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:05,104 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:05,104 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:05,104 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:05,109 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073741995_1171 (size=12001) 2024-11-20T09:00:05,309 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:05,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48420 deadline: 1732093265307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:05,310 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:05,310 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:05,310 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:05,320 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:05,320 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:05,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48394 deadline: 1732093265318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:05,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48450 deadline: 1732093265318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:05,322 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:05,322 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:05,322 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:05,322 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:05,323 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:05,323 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:05,511 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=122 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/C/9d0779fadc434aaf868ac25f9fcb945a 2024-11-20T09:00:05,516 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/A/c04675b5ee5b4bd3b555cb77a691f5d1 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/A/c04675b5ee5b4bd3b555cb77a691f5d1 2024-11-20T09:00:05,523 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/A/c04675b5ee5b4bd3b555cb77a691f5d1, entries=200, sequenceid=122, filesize=38.6 K 2024-11-20T09:00:05,524 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/B/b09224912527476ca924e92bc11b8a3f as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/B/b09224912527476ca924e92bc11b8a3f 2024-11-20T09:00:05,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,530 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/B/b09224912527476ca924e92bc11b8a3f, entries=150, sequenceid=122, filesize=11.7 K 2024-11-20T09:00:05,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,531 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/C/9d0779fadc434aaf868ac25f9fcb945a as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/C/9d0779fadc434aaf868ac25f9fcb945a 2024-11-20T09:00:05,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,538 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/C/9d0779fadc434aaf868ac25f9fcb945a, entries=150, sequenceid=122, filesize=11.7 K 2024-11-20T09:00:05,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,539 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=40.25 KB/41220 for f0e725d6c797c04eb86dfc3be87ab3ba in 568ms, sequenceid=122, compaction requested=true 2024-11-20T09:00:05,539 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f0e725d6c797c04eb86dfc3be87ab3ba: 2024-11-20T09:00:05,539 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f0e725d6c797c04eb86dfc3be87ab3ba:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T09:00:05,539 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T09:00:05,539 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T09:00:05,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,539 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T09:00:05,540 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f0e725d6c797c04eb86dfc3be87ab3ba:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T09:00:05,540 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T09:00:05,540 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f0e725d6c797c04eb86dfc3be87ab3ba:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T09:00:05,540 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T09:00:05,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,541 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T09:00:05,541 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 149701 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T09:00:05,541 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1541): f0e725d6c797c04eb86dfc3be87ab3ba/B is initiating minor compaction (all files) 2024-11-20T09:00:05,541 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HStore(1541): f0e725d6c797c04eb86dfc3be87ab3ba/A is initiating minor compaction (all files) 2024-11-20T09:00:05,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,541 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f0e725d6c797c04eb86dfc3be87ab3ba/B in TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. 2024-11-20T09:00:05,541 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f0e725d6c797c04eb86dfc3be87ab3ba/A in TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. 2024-11-20T09:00:05,542 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/A/2f744b70760844c9ac03430f5eeeedc4, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/A/6a279628dca84f46b8b6eddb02457932, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/A/ef9a68d183ea47b5a6356aa1384ee3e8, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/A/c04675b5ee5b4bd3b555cb77a691f5d1] into tmpdir=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp, totalSize=146.2 K 2024-11-20T09:00:05,542 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/B/2dab945a9c664ebb88f30ee634af33a5, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/B/4ee36236777344abba8cb58c74f4f910, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/B/45abc1a01d1341d1a67cfe1881a3a929, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/B/b09224912527476ca924e92bc11b8a3f] into tmpdir=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp, totalSize=47.0 K 2024-11-20T09:00:05,542 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. 2024-11-20T09:00:05,542 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. files: [hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/A/2f744b70760844c9ac03430f5eeeedc4, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/A/6a279628dca84f46b8b6eddb02457932, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/A/ef9a68d183ea47b5a6356aa1384ee3e8, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/A/c04675b5ee5b4bd3b555cb77a691f5d1] 2024-11-20T09:00:05,542 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting 2dab945a9c664ebb88f30ee634af33a5, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1732093202212 2024-11-20T09:00:05,542 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2f744b70760844c9ac03430f5eeeedc4, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1732093202212 2024-11-20T09:00:05,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,543 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting 4ee36236777344abba8cb58c74f4f910, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1732093202925 2024-11-20T09:00:05,543 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6a279628dca84f46b8b6eddb02457932, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1732093202925 2024-11-20T09:00:05,543 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting 45abc1a01d1341d1a67cfe1881a3a929, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1732093203599 2024-11-20T09:00:05,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,543 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.Compactor(225): Compacting ef9a68d183ea47b5a6356aa1384ee3e8, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1732093203599 2024-11-20T09:00:05,543 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting b09224912527476ca924e92bc11b8a3f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=122, earliestPutTs=1732093204846 2024-11-20T09:00:05,544 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.Compactor(225): Compacting c04675b5ee5b4bd3b555cb77a691f5d1, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=122, earliestPutTs=1732093204833 2024-11-20T09:00:05,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,556 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=f0e725d6c797c04eb86dfc3be87ab3ba] 2024-11-20T09:00:05,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,571 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241120055debfc7bd14e198a53de64986c0287_f0e725d6c797c04eb86dfc3be87ab3ba store=[table=TestAcidGuarantees family=A region=f0e725d6c797c04eb86dfc3be87ab3ba] 2024-11-20T09:00:05,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-20T09:00:05,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,577 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T09:00:05,578 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T09:00:05,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,579 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f0e725d6c797c04eb86dfc3be87ab3ba#B#compaction#150 average throughput is 0.82 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-20T09:00:05,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] procedure2.ProcedureExecutor(1139): Stored pid=38, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees 2024-11-20T09:00:05,580 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/B/5574eb6207f640a7bfa94653e09c9bc8 is 50, key is test_row_0/B:col10/1732093204969/Put/seqid=0 2024-11-20T09:00:05,580 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=38, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T09:00:05,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-20T09:00:05,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,581 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=38, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T09:00:05,581 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=38, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T09:00:05,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,584 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241120055debfc7bd14e198a53de64986c0287_f0e725d6c797c04eb86dfc3be87ab3ba, store=[table=TestAcidGuarantees family=A region=f0e725d6c797c04eb86dfc3be87ab3ba] 2024-11-20T09:00:05,584 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120055debfc7bd14e198a53de64986c0287_f0e725d6c797c04eb86dfc3be87ab3ba because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=f0e725d6c797c04eb86dfc3be87ab3ba] 2024-11-20T09:00:05,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,605 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073741996_1172 (size=12241) 2024-11-20T09:00:05,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,623 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073741997_1173 (size=4469) 2024-11-20T09:00:05,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(8855): Flush requested on f0e725d6c797c04eb86dfc3be87ab3ba 2024-11-20T09:00:05,636 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T09:00:05,637 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f0e725d6c797c04eb86dfc3be87ab3ba 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-20T09:00:05,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,646 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112013a4bed7123e4ab0829eefdb9f90dc6c_f0e725d6c797c04eb86dfc3be87ab3ba is 50, key is test_row_0/A:col10/1732093205637/Put/seqid=0 2024-11-20T09:00:05,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,681 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073741998_1174 (size=17234) 2024-11-20T09:00:05,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-20T09:00:05,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,692 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,699 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112013a4bed7123e4ab0829eefdb9f90dc6c_f0e725d6c797c04eb86dfc3be87ab3ba to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112013a4bed7123e4ab0829eefdb9f90dc6c_f0e725d6c797c04eb86dfc3be87ab3ba 2024-11-20T09:00:05,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,701 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/A/1b8f2a842cc14122b7a45b0c725df0b2, store: [table=TestAcidGuarantees family=A region=f0e725d6c797c04eb86dfc3be87ab3ba] 2024-11-20T09:00:05,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,702 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/A/1b8f2a842cc14122b7a45b0c725df0b2 is 175, key is test_row_0/A:col10/1732093205637/Put/seqid=0 2024-11-20T09:00:05,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,710 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073741999_1175 (size=48335) 2024-11-20T09:00:05,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,711 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=134, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/A/1b8f2a842cc14122b7a45b0c725df0b2 2024-11-20T09:00:05,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,725 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/B/502d9dffa5284da09e81e206326a48dd is 50, key is test_row_0/B:col10/1732093205637/Put/seqid=0 2024-11-20T09:00:05,735 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-20T09:00:05,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. 2024-11-20T09:00:05,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. as already flushing 2024-11-20T09:00:05,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. 2024-11-20T09:00:05,735 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => f0e725d6c797c04eb86dfc3be87ab3ba, NAME => 'TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:05,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => f0e725d6c797c04eb86dfc3be87ab3ba, NAME => 'TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:05,736 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:05,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48450 deadline: 1732093265729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:05,737 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:05,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0e725d6c797c04eb86dfc3be87ab3ba, NAME => 'TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0e725d6c797c04eb86dfc3be87ab3ba, NAME => 'TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:05,737 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:05,737 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:05,743 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:05,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48466 deadline: 1732093265731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:05,743 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:05,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48394 deadline: 1732093265733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:05,744 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:05,744 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:05,744 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:05,744 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:05,744 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:05,745 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:05,747 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:05,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48420 deadline: 1732093265735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:05,747 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:05,747 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:05,747 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:05,747 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:05,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48388 deadline: 1732093265736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:05,748 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:05,748 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:05,748 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:05,755 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742000_1176 (size=9757) 2024-11-20T09:00:05,756 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=134 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/B/502d9dffa5284da09e81e206326a48dd 2024-11-20T09:00:05,767 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/C/8bb08d6d9e324cf0b49265c8b5e62257 is 50, key is test_row_0/C:col10/1732093205637/Put/seqid=0 2024-11-20T09:00:05,778 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742001_1177 (size=9757) 2024-11-20T09:00:05,779 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=134 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/C/8bb08d6d9e324cf0b49265c8b5e62257 2024-11-20T09:00:05,785 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/A/1b8f2a842cc14122b7a45b0c725df0b2 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/A/1b8f2a842cc14122b7a45b0c725df0b2 2024-11-20T09:00:05,793 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/A/1b8f2a842cc14122b7a45b0c725df0b2, entries=250, sequenceid=134, filesize=47.2 K 2024-11-20T09:00:05,794 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/B/502d9dffa5284da09e81e206326a48dd as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/B/502d9dffa5284da09e81e206326a48dd 2024-11-20T09:00:05,799 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/B/502d9dffa5284da09e81e206326a48dd, entries=100, sequenceid=134, filesize=9.5 K 2024-11-20T09:00:05,800 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/C/8bb08d6d9e324cf0b49265c8b5e62257 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/C/8bb08d6d9e324cf0b49265c8b5e62257 2024-11-20T09:00:05,806 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/C/8bb08d6d9e324cf0b49265c8b5e62257, entries=100, sequenceid=134, filesize=9.5 K 2024-11-20T09:00:05,807 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for f0e725d6c797c04eb86dfc3be87ab3ba in 171ms, sequenceid=134, compaction requested=true 2024-11-20T09:00:05,807 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f0e725d6c797c04eb86dfc3be87ab3ba: 2024-11-20T09:00:05,807 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f0e725d6c797c04eb86dfc3be87ab3ba:A, priority=-2147483648, current under compaction store size is 3 2024-11-20T09:00:05,807 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-20T09:00:05,807 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f0e725d6c797c04eb86dfc3be87ab3ba:B, priority=-2147483648, current under compaction store size is 3 2024-11-20T09:00:05,807 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-20T09:00:05,807 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f0e725d6c797c04eb86dfc3be87ab3ba:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T09:00:05,807 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=4), splitQueue=0 2024-11-20T09:00:05,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(8855): Flush requested on f0e725d6c797c04eb86dfc3be87ab3ba 2024-11-20T09:00:05,851 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T09:00:05,851 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f0e725d6c797c04eb86dfc3be87ab3ba 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-20T09:00:05,870 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:05,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48394 deadline: 1732093265862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:05,870 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:05,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48466 deadline: 1732093265864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:05,871 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:05,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48420 deadline: 1732093265866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:05,872 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:05,872 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:05,872 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:05,872 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:05,872 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:05,872 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:05,872 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:05,872 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:05,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48388 deadline: 1732093265867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:05,873 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:05,873 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:05,873 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:05,873 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:05,873 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:05,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48450 deadline: 1732093265868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:05,873 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:05,873 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:05,873 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:05,874 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:05,885 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120614a9b6824b5479986171cd202c1cda0_f0e725d6c797c04eb86dfc3be87ab3ba is 50, key is test_row_0/A:col10/1732093205726/Put/seqid=0 2024-11-20T09:00:05,889 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-20T09:00:05,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. 2024-11-20T09:00:05,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. as already flushing 2024-11-20T09:00:05,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. 2024-11-20T09:00:05,890 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => f0e725d6c797c04eb86dfc3be87ab3ba, NAME => 'TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:05,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => f0e725d6c797c04eb86dfc3be87ab3ba, NAME => 'TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:05,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0e725d6c797c04eb86dfc3be87ab3ba, NAME => 'TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0e725d6c797c04eb86dfc3be87ab3ba, NAME => 'TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:05,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-20T09:00:05,902 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742002_1178 (size=17284) 2024-11-20T09:00:05,905 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:05,909 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120614a9b6824b5479986171cd202c1cda0_f0e725d6c797c04eb86dfc3be87ab3ba to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120614a9b6824b5479986171cd202c1cda0_f0e725d6c797c04eb86dfc3be87ab3ba 2024-11-20T09:00:05,910 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/A/d3a402d8d3604d11ade73942b38c8c00, store: [table=TestAcidGuarantees family=A region=f0e725d6c797c04eb86dfc3be87ab3ba] 2024-11-20T09:00:05,911 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/A/d3a402d8d3604d11ade73942b38c8c00 is 175, key is test_row_0/A:col10/1732093205726/Put/seqid=0 2024-11-20T09:00:05,921 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742003_1179 (size=48389) 2024-11-20T09:00:05,922 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=160, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/A/d3a402d8d3604d11ade73942b38c8c00 2024-11-20T09:00:05,934 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/B/367116f84c0f41eaae074366162013f6 is 50, key is test_row_0/B:col10/1732093205726/Put/seqid=0 2024-11-20T09:00:05,950 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742004_1180 (size=12151) 2024-11-20T09:00:05,950 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=160 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/B/367116f84c0f41eaae074366162013f6 2024-11-20T09:00:05,968 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/C/1190d219706e44279cb3f77e116de28b is 50, key is test_row_0/C:col10/1732093205726/Put/seqid=0 2024-11-20T09:00:05,981 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:05,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48466 deadline: 1732093265977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:05,981 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:05,981 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:05,981 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:05,982 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:05,982 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:05,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48394 deadline: 1732093265977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:05,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48420 deadline: 1732093265977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:05,982 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:05,982 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:05,982 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:05,982 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:05,982 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:05,982 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:05,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48450 deadline: 1732093265978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:05,982 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:05,982 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:05,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48388 deadline: 1732093265981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:05,983 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:05,983 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:05,983 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:05,985 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742005_1181 (size=12151) 2024-11-20T09:00:05,985 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:05,986 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:05,986 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:05,986 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=160 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/C/1190d219706e44279cb3f77e116de28b 2024-11-20T09:00:05,993 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/A/d3a402d8d3604d11ade73942b38c8c00 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/A/d3a402d8d3604d11ade73942b38c8c00 2024-11-20T09:00:05,998 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/A/d3a402d8d3604d11ade73942b38c8c00, entries=250, sequenceid=160, filesize=47.3 K 2024-11-20T09:00:05,999 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/B/367116f84c0f41eaae074366162013f6 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/B/367116f84c0f41eaae074366162013f6 2024-11-20T09:00:06,011 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/B/367116f84c0f41eaae074366162013f6, entries=150, sequenceid=160, filesize=11.9 K 2024-11-20T09:00:06,011 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/B/5574eb6207f640a7bfa94653e09c9bc8 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/B/5574eb6207f640a7bfa94653e09c9bc8 2024-11-20T09:00:06,014 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/C/1190d219706e44279cb3f77e116de28b as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/C/1190d219706e44279cb3f77e116de28b 2024-11-20T09:00:06,018 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f0e725d6c797c04eb86dfc3be87ab3ba/B of f0e725d6c797c04eb86dfc3be87ab3ba into 5574eb6207f640a7bfa94653e09c9bc8(size=12.0 K), total size for store is 33.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T09:00:06,018 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f0e725d6c797c04eb86dfc3be87ab3ba: 2024-11-20T09:00:06,018 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., storeName=f0e725d6c797c04eb86dfc3be87ab3ba/B, priority=12, startTime=1732093205539; duration=0sec 2024-11-20T09:00:06,019 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=4), splitQueue=0 2024-11-20T09:00:06,019 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f0e725d6c797c04eb86dfc3be87ab3ba:B 2024-11-20T09:00:06,019 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 6 store files, 4 compacting, 2 eligible, 16 blocking 2024-11-20T09:00:06,020 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-20T09:00:06,020 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-20T09:00:06,020 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. because compaction request was cancelled 2024-11-20T09:00:06,020 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f0e725d6c797c04eb86dfc3be87ab3ba:A 2024-11-20T09:00:06,020 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-20T09:00:06,022 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 57864 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-20T09:00:06,022 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1541): f0e725d6c797c04eb86dfc3be87ab3ba/C is initiating minor compaction (all files) 2024-11-20T09:00:06,022 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f0e725d6c797c04eb86dfc3be87ab3ba/C in TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. 2024-11-20T09:00:06,023 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/C/3126051b8ac2415593113cc36e85695f, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/C/0377e3b27ed74c2b8aa8a87dd9121431, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/C/04fb40e09f9f4dfca658cfc049179d49, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/C/9d0779fadc434aaf868ac25f9fcb945a, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/C/8bb08d6d9e324cf0b49265c8b5e62257] into tmpdir=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp, totalSize=56.5 K 2024-11-20T09:00:06,023 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting 3126051b8ac2415593113cc36e85695f, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1732093202212 2024-11-20T09:00:06,023 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/C/1190d219706e44279cb3f77e116de28b, entries=150, sequenceid=160, filesize=11.9 K 2024-11-20T09:00:06,024 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting 0377e3b27ed74c2b8aa8a87dd9121431, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1732093202925 2024-11-20T09:00:06,025 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting 04fb40e09f9f4dfca658cfc049179d49, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1732093203599 2024-11-20T09:00:06,025 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting 9d0779fadc434aaf868ac25f9fcb945a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=122, earliestPutTs=1732093204846 2024-11-20T09:00:06,025 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f0e725d6c797c04eb86dfc3be87ab3ba#A#compaction#149 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T09:00:06,026 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting 8bb08d6d9e324cf0b49265c8b5e62257, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1732093204982 2024-11-20T09:00:06,026 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/A/0ff292d9febc47b896283171fd4ceb4e is 175, key is test_row_0/A:col10/1732093204969/Put/seqid=0 2024-11-20T09:00:06,029 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for f0e725d6c797c04eb86dfc3be87ab3ba in 177ms, sequenceid=160, compaction requested=false 2024-11-20T09:00:06,029 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f0e725d6c797c04eb86dfc3be87ab3ba: 2024-11-20T09:00:06,043 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-20T09:00:06,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. 2024-11-20T09:00:06,043 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2902): Flushing f0e725d6c797c04eb86dfc3be87ab3ba 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-20T09:00:06,055 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742006_1182 (size=31195) 2024-11-20T09:00:06,059 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f0e725d6c797c04eb86dfc3be87ab3ba#C#compaction#157 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T09:00:06,060 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/C/7cad10a70d7b402381ce9c9baccd6e65 is 50, key is test_row_0/C:col10/1732093205637/Put/seqid=0 2024-11-20T09:00:06,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411209aca9bdb2ebe4142977109b6a8305496_f0e725d6c797c04eb86dfc3be87ab3ba is 50, key is test_row_0/A:col10/1732093205862/Put/seqid=0 2024-11-20T09:00:06,098 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742007_1183 (size=12375) 2024-11-20T09:00:06,108 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/C/7cad10a70d7b402381ce9c9baccd6e65 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/C/7cad10a70d7b402381ce9c9baccd6e65 2024-11-20T09:00:06,118 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in f0e725d6c797c04eb86dfc3be87ab3ba/C of f0e725d6c797c04eb86dfc3be87ab3ba into 7cad10a70d7b402381ce9c9baccd6e65(size=12.1 K), total size for store is 24.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T09:00:06,118 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f0e725d6c797c04eb86dfc3be87ab3ba: 2024-11-20T09:00:06,119 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., storeName=f0e725d6c797c04eb86dfc3be87ab3ba/C, priority=11, startTime=1732093205807; duration=0sec 2024-11-20T09:00:06,119 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-20T09:00:06,119 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f0e725d6c797c04eb86dfc3be87ab3ba:C 2024-11-20T09:00:06,119 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f0e725d6c797c04eb86dfc3be87ab3ba:B 2024-11-20T09:00:06,119 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f0e725d6c797c04eb86dfc3be87ab3ba:C 2024-11-20T09:00:06,119 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T09:00:06,122 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34149 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T09:00:06,122 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1541): f0e725d6c797c04eb86dfc3be87ab3ba/B is initiating minor compaction (all files) 2024-11-20T09:00:06,122 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f0e725d6c797c04eb86dfc3be87ab3ba/B in TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. 2024-11-20T09:00:06,122 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/B/5574eb6207f640a7bfa94653e09c9bc8, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/B/502d9dffa5284da09e81e206326a48dd, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/B/367116f84c0f41eaae074366162013f6] into tmpdir=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp, totalSize=33.3 K 2024-11-20T09:00:06,123 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting 5574eb6207f640a7bfa94653e09c9bc8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=122, earliestPutTs=1732093204846 2024-11-20T09:00:06,123 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting 502d9dffa5284da09e81e206326a48dd, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1732093204982 2024-11-20T09:00:06,124 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting 367116f84c0f41eaae074366162013f6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=160, earliestPutTs=1732093205726 2024-11-20T09:00:06,128 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742008_1184 (size=12304) 2024-11-20T09:00:06,142 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f0e725d6c797c04eb86dfc3be87ab3ba#B#compaction#159 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T09:00:06,143 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/B/92c77a6421bf4080b952e764d72b309e is 50, key is test_row_0/B:col10/1732093205726/Put/seqid=0 2024-11-20T09:00:06,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742009_1185 (size=12493) 2024-11-20T09:00:06,166 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/B/92c77a6421bf4080b952e764d72b309e as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/B/92c77a6421bf4080b952e764d72b309e 2024-11-20T09:00:06,172 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f0e725d6c797c04eb86dfc3be87ab3ba/B of f0e725d6c797c04eb86dfc3be87ab3ba into 92c77a6421bf4080b952e764d72b309e(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T09:00:06,172 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f0e725d6c797c04eb86dfc3be87ab3ba: 2024-11-20T09:00:06,173 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., storeName=f0e725d6c797c04eb86dfc3be87ab3ba/B, priority=13, startTime=1732093205807; duration=0sec 2024-11-20T09:00:06,173 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T09:00:06,173 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f0e725d6c797c04eb86dfc3be87ab3ba:B 2024-11-20T09:00:06,174 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-20T09:00:06,176 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-20T09:00:06,176 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-20T09:00:06,176 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. because compaction request was cancelled 2024-11-20T09:00:06,176 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f0e725d6c797c04eb86dfc3be87ab3ba:C 2024-11-20T09:00:06,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(8855): Flush requested on f0e725d6c797c04eb86dfc3be87ab3ba 2024-11-20T09:00:06,191 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. as already flushing 2024-11-20T09:00:06,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-20T09:00:06,223 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:06,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48420 deadline: 1732093266217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:06,224 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:06,224 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:06,224 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:06,228 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:06,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48450 deadline: 1732093266219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:06,228 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:06,228 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:06,228 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:06,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48388 deadline: 1732093266220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:06,228 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:06,229 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:06,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48394 deadline: 1732093266221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:06,229 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:06,229 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:06,229 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:06,229 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:06,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48466 deadline: 1732093266223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:06,229 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:06,229 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:06,229 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:06,229 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:06,230 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:06,230 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:06,336 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:06,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48420 deadline: 1732093266327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:06,336 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:06,336 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:06,337 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:06,338 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:06,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48394 deadline: 1732093266337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:06,338 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:06,338 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:06,338 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:06,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48450 deadline: 1732093266337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:06,339 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:06,339 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:06,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48466 deadline: 1732093266337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:06,339 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:06,339 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:06,339 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:06,339 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:06,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48388 deadline: 1732093266338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:06,339 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:06,340 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:06,340 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:06,342 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:06,342 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:06,342 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:06,462 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/A/0ff292d9febc47b896283171fd4ceb4e as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/A/0ff292d9febc47b896283171fd4ceb4e 2024-11-20T09:00:06,475 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f0e725d6c797c04eb86dfc3be87ab3ba/A of f0e725d6c797c04eb86dfc3be87ab3ba into 0ff292d9febc47b896283171fd4ceb4e(size=30.5 K), total size for store is 124.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T09:00:06,475 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f0e725d6c797c04eb86dfc3be87ab3ba: 2024-11-20T09:00:06,475 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., storeName=f0e725d6c797c04eb86dfc3be87ab3ba/A, priority=12, startTime=1732093205539; duration=0sec 2024-11-20T09:00:06,476 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T09:00:06,476 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f0e725d6c797c04eb86dfc3be87ab3ba:A 2024-11-20T09:00:06,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:06,532 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411209aca9bdb2ebe4142977109b6a8305496_f0e725d6c797c04eb86dfc3be87ab3ba to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411209aca9bdb2ebe4142977109b6a8305496_f0e725d6c797c04eb86dfc3be87ab3ba 2024-11-20T09:00:06,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/A/bc898d3b742c42e79dc1af0abba0316c, store: [table=TestAcidGuarantees family=A region=f0e725d6c797c04eb86dfc3be87ab3ba] 2024-11-20T09:00:06,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/A/bc898d3b742c42e79dc1af0abba0316c is 175, key is test_row_0/A:col10/1732093205862/Put/seqid=0 2024-11-20T09:00:06,539 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742010_1186 (size=31105) 2024-11-20T09:00:06,540 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=171, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/A/bc898d3b742c42e79dc1af0abba0316c 2024-11-20T09:00:06,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/B/204bc2d3789d4ce2b16e451a8be472fa is 50, key is test_row_0/B:col10/1732093205862/Put/seqid=0 2024-11-20T09:00:06,549 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:06,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48420 deadline: 1732093266547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:06,549 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:06,549 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:06,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48450 deadline: 1732093266547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:06,549 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:06,550 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:06,550 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:06,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48394 deadline: 1732093266547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:06,550 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:06,550 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:06,550 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:06,550 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:06,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48466 deadline: 1732093266548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:06,550 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:06,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48388 deadline: 1732093266548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:06,552 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:06,552 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:06,552 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:06,552 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:06,552 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:06,552 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:06,552 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:06,552 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:06,552 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:06,557 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742011_1187 (size=12151) 2024-11-20T09:00:06,563 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=171 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/B/204bc2d3789d4ce2b16e451a8be472fa 2024-11-20T09:00:06,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/C/b5ea089291a74d1ba31b13d0ce3c7b92 is 50, key is test_row_0/C:col10/1732093205862/Put/seqid=0 2024-11-20T09:00:06,584 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742012_1188 (size=12151) 2024-11-20T09:00:06,589 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=171 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/C/b5ea089291a74d1ba31b13d0ce3c7b92 2024-11-20T09:00:06,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/A/bc898d3b742c42e79dc1af0abba0316c as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/A/bc898d3b742c42e79dc1af0abba0316c 2024-11-20T09:00:06,605 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/A/bc898d3b742c42e79dc1af0abba0316c, entries=150, sequenceid=171, filesize=30.4 K 2024-11-20T09:00:06,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/B/204bc2d3789d4ce2b16e451a8be472fa as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/B/204bc2d3789d4ce2b16e451a8be472fa 2024-11-20T09:00:06,613 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/B/204bc2d3789d4ce2b16e451a8be472fa, entries=150, sequenceid=171, filesize=11.9 K 2024-11-20T09:00:06,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/C/b5ea089291a74d1ba31b13d0ce3c7b92 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/C/b5ea089291a74d1ba31b13d0ce3c7b92 2024-11-20T09:00:06,622 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/C/b5ea089291a74d1ba31b13d0ce3c7b92, entries=150, sequenceid=171, filesize=11.9 K 2024-11-20T09:00:06,623 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=154.31 KB/158010 for f0e725d6c797c04eb86dfc3be87ab3ba in 580ms, sequenceid=171, compaction requested=true 2024-11-20T09:00:06,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2603): Flush status journal for f0e725d6c797c04eb86dfc3be87ab3ba: 2024-11-20T09:00:06,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. 2024-11-20T09:00:06,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=39 2024-11-20T09:00:06,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.HMaster(4169): Remote procedure done, pid=39 2024-11-20T09:00:06,627 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=38 2024-11-20T09:00:06,627 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=38, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0440 sec 2024-11-20T09:00:06,630 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees in 1.0510 sec 2024-11-20T09:00:06,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-20T09:00:06,717 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T09:00:06,717 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T09:00:06,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] procedure2.ProcedureExecutor(1139): Stored pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees 2024-11-20T09:00:06,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-20T09:00:06,720 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T09:00:06,721 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T09:00:06,721 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=40, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T09:00:06,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-20T09:00:06,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(8855): Flush requested on f0e725d6c797c04eb86dfc3be87ab3ba 2024-11-20T09:00:06,862 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T09:00:06,862 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f0e725d6c797c04eb86dfc3be87ab3ba 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-20T09:00:06,874 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120e94abf5d9d5f4388beb7d4cafcd74124_f0e725d6c797c04eb86dfc3be87ab3ba is 50, key is test_row_0/A:col10/1732093206860/Put/seqid=0 2024-11-20T09:00:06,875 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-20T09:00:06,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. 2024-11-20T09:00:06,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. as already flushing 2024-11-20T09:00:06,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. 2024-11-20T09:00:06,876 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => f0e725d6c797c04eb86dfc3be87ab3ba, NAME => 'TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:06,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => f0e725d6c797c04eb86dfc3be87ab3ba, NAME => 'TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:06,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0e725d6c797c04eb86dfc3be87ab3ba, NAME => 'TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0e725d6c797c04eb86dfc3be87ab3ba, NAME => 'TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:06,893 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742013_1189 (size=14794) 2024-11-20T09:00:06,908 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:06,908 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:06,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48466 deadline: 1732093266868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:06,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48394 deadline: 1732093266869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:06,909 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:06,909 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:06,909 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:06,909 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:06,909 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:06,909 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:06,913 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:06,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48420 deadline: 1732093266909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:06,914 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:06,914 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:06,914 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:06,914 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:06,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48388 deadline: 1732093266909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:06,914 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:06,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48450 deadline: 1732093266910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:06,914 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:06,914 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:06,914 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:06,915 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:06,915 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:06,915 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:07,017 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:07,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48394 deadline: 1732093267017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:07,018 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:07,018 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:07,018 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:07,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48450 deadline: 1732093267017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:07,018 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:07,018 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:07,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48420 deadline: 1732093267017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:07,019 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:07,019 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:07,019 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:07,019 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:07,019 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:07,019 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:07,019 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:07,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48466 deadline: 1732093267018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:07,020 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:07,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48388 deadline: 1732093267019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:07,020 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:07,020 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:07,020 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:07,021 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:07,021 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:07,021 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:07,029 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-20T09:00:07,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. 2024-11-20T09:00:07,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. as already flushing 2024-11-20T09:00:07,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. 2024-11-20T09:00:07,029 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => f0e725d6c797c04eb86dfc3be87ab3ba, NAME => 'TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:07,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => f0e725d6c797c04eb86dfc3be87ab3ba, NAME => 'TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:07,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0e725d6c797c04eb86dfc3be87ab3ba, NAME => 'TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0e725d6c797c04eb86dfc3be87ab3ba, NAME => 'TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:07,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-20T09:00:07,189 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-20T09:00:07,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. 2024-11-20T09:00:07,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. as already flushing 2024-11-20T09:00:07,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. 2024-11-20T09:00:07,189 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => f0e725d6c797c04eb86dfc3be87ab3ba, NAME => 'TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:07,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => f0e725d6c797c04eb86dfc3be87ab3ba, NAME => 'TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:07,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0e725d6c797c04eb86dfc3be87ab3ba, NAME => 'TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0e725d6c797c04eb86dfc3be87ab3ba, NAME => 'TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:07,231 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:07,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48394 deadline: 1732093267227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:07,232 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:07,232 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:07,232 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:07,232 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:07,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48450 deadline: 1732093267227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:07,232 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:07,233 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:07,233 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:07,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48420 deadline: 1732093267227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:07,233 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:07,233 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:07,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48466 deadline: 1732093267227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:07,233 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:07,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48388 deadline: 1732093267229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:07,234 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:07,234 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:07,234 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:07,234 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:07,234 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:07,234 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:07,235 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:07,235 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:07,235 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:07,295 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:07,300 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120e94abf5d9d5f4388beb7d4cafcd74124_f0e725d6c797c04eb86dfc3be87ab3ba to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120e94abf5d9d5f4388beb7d4cafcd74124_f0e725d6c797c04eb86dfc3be87ab3ba 2024-11-20T09:00:07,301 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/A/be529b6eb2b54b3eae6d8c04d74ab6b1, store: [table=TestAcidGuarantees family=A region=f0e725d6c797c04eb86dfc3be87ab3ba] 2024-11-20T09:00:07,301 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/A/be529b6eb2b54b3eae6d8c04d74ab6b1 is 175, key is test_row_0/A:col10/1732093206860/Put/seqid=0 2024-11-20T09:00:07,306 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742014_1190 (size=39749) 2024-11-20T09:00:07,306 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=201, memsize=53.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/A/be529b6eb2b54b3eae6d8c04d74ab6b1 2024-11-20T09:00:07,315 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/B/9a6a79bb77114833840f31896f9a97cc is 50, key is test_row_0/B:col10/1732093206860/Put/seqid=0 2024-11-20T09:00:07,346 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742015_1191 (size=12151) 2024-11-20T09:00:07,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-20T09:00:07,363 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-20T09:00:07,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. 2024-11-20T09:00:07,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. as already flushing 2024-11-20T09:00:07,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. 2024-11-20T09:00:07,364 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => f0e725d6c797c04eb86dfc3be87ab3ba, NAME => 'TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:07,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => f0e725d6c797c04eb86dfc3be87ab3ba, NAME => 'TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:07,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0e725d6c797c04eb86dfc3be87ab3ba, NAME => 'TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0e725d6c797c04eb86dfc3be87ab3ba, NAME => 'TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:07,517 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-20T09:00:07,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. 2024-11-20T09:00:07,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. as already flushing 2024-11-20T09:00:07,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. 2024-11-20T09:00:07,517 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => f0e725d6c797c04eb86dfc3be87ab3ba, NAME => 'TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:07,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => f0e725d6c797c04eb86dfc3be87ab3ba, NAME => 'TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:07,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0e725d6c797c04eb86dfc3be87ab3ba, NAME => 'TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0e725d6c797c04eb86dfc3be87ab3ba, NAME => 'TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:07,540 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:07,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48394 deadline: 1732093267537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:07,541 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:07,541 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:07,541 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:07,541 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:07,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48450 deadline: 1732093267537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:07,542 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:07,542 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:07,542 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:07,547 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:07,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48388 deadline: 1732093267547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:07,548 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:07,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48466 deadline: 1732093267547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:07,549 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:07,549 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:07,549 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:07,549 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:07,549 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:07,549 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:07,549 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:07,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48420 deadline: 1732093267547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:07,549 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:07,549 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:07,550 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:07,671 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-20T09:00:07,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. 2024-11-20T09:00:07,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. as already flushing 2024-11-20T09:00:07,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. 2024-11-20T09:00:07,672 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => f0e725d6c797c04eb86dfc3be87ab3ba, NAME => 'TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:07,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => f0e725d6c797c04eb86dfc3be87ab3ba, NAME => 'TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:07,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0e725d6c797c04eb86dfc3be87ab3ba, NAME => 'TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0e725d6c797c04eb86dfc3be87ab3ba, NAME => 'TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:07,746 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=201 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/B/9a6a79bb77114833840f31896f9a97cc 2024-11-20T09:00:07,757 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/C/bcf5b04ce6fc4895a880cb7852a4a3dc is 50, key is test_row_0/C:col10/1732093206860/Put/seqid=0 2024-11-20T09:00:07,765 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742016_1192 (size=12151) 2024-11-20T09:00:07,765 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=201 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/C/bcf5b04ce6fc4895a880cb7852a4a3dc 2024-11-20T09:00:07,773 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/A/be529b6eb2b54b3eae6d8c04d74ab6b1 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/A/be529b6eb2b54b3eae6d8c04d74ab6b1 2024-11-20T09:00:07,777 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/A/be529b6eb2b54b3eae6d8c04d74ab6b1, entries=200, sequenceid=201, filesize=38.8 K 2024-11-20T09:00:07,788 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/B/9a6a79bb77114833840f31896f9a97cc as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/B/9a6a79bb77114833840f31896f9a97cc 2024-11-20T09:00:07,794 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/B/9a6a79bb77114833840f31896f9a97cc, entries=150, sequenceid=201, filesize=11.9 K 2024-11-20T09:00:07,795 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/C/bcf5b04ce6fc4895a880cb7852a4a3dc as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/C/bcf5b04ce6fc4895a880cb7852a4a3dc 2024-11-20T09:00:07,799 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/C/bcf5b04ce6fc4895a880cb7852a4a3dc, entries=150, sequenceid=201, filesize=11.9 K 2024-11-20T09:00:07,801 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=53.67 KB/54960 for f0e725d6c797c04eb86dfc3be87ab3ba in 939ms, sequenceid=201, compaction requested=true 2024-11-20T09:00:07,801 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f0e725d6c797c04eb86dfc3be87ab3ba: 2024-11-20T09:00:07,801 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-20T09:00:07,802 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f0e725d6c797c04eb86dfc3be87ab3ba:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T09:00:07,802 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T09:00:07,802 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T09:00:07,803 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T09:00:07,803 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1541): f0e725d6c797c04eb86dfc3be87ab3ba/B is initiating minor compaction (all files) 2024-11-20T09:00:07,803 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f0e725d6c797c04eb86dfc3be87ab3ba/B in TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. 2024-11-20T09:00:07,803 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/B/92c77a6421bf4080b952e764d72b309e, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/B/204bc2d3789d4ce2b16e451a8be472fa, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/B/9a6a79bb77114833840f31896f9a97cc] into tmpdir=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp, totalSize=35.9 K 2024-11-20T09:00:07,804 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 198773 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-20T09:00:07,804 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HStore(1541): f0e725d6c797c04eb86dfc3be87ab3ba/A is initiating minor compaction (all files) 2024-11-20T09:00:07,804 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f0e725d6c797c04eb86dfc3be87ab3ba/A in TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. 2024-11-20T09:00:07,804 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/A/0ff292d9febc47b896283171fd4ceb4e, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/A/1b8f2a842cc14122b7a45b0c725df0b2, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/A/d3a402d8d3604d11ade73942b38c8c00, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/A/bc898d3b742c42e79dc1af0abba0316c, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/A/be529b6eb2b54b3eae6d8c04d74ab6b1] into tmpdir=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp, totalSize=194.1 K 2024-11-20T09:00:07,804 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=11 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. 2024-11-20T09:00:07,804 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. files: [hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/A/0ff292d9febc47b896283171fd4ceb4e, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/A/1b8f2a842cc14122b7a45b0c725df0b2, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/A/d3a402d8d3604d11ade73942b38c8c00, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/A/bc898d3b742c42e79dc1af0abba0316c, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/A/be529b6eb2b54b3eae6d8c04d74ab6b1] 2024-11-20T09:00:07,805 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f0e725d6c797c04eb86dfc3be87ab3ba:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T09:00:07,805 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T09:00:07,805 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f0e725d6c797c04eb86dfc3be87ab3ba:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T09:00:07,805 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T09:00:07,805 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting 92c77a6421bf4080b952e764d72b309e, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=160, earliestPutTs=1732093205726 2024-11-20T09:00:07,805 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0ff292d9febc47b896283171fd4ceb4e, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=122, earliestPutTs=1732093204846 2024-11-20T09:00:07,805 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting 204bc2d3789d4ce2b16e451a8be472fa, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1732093205862 2024-11-20T09:00:07,805 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1b8f2a842cc14122b7a45b0c725df0b2, keycount=250, bloomtype=ROW, size=47.2 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1732093204979 2024-11-20T09:00:07,806 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.Compactor(225): Compacting d3a402d8d3604d11ade73942b38c8c00, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=160, earliestPutTs=1732093205726 2024-11-20T09:00:07,806 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting 9a6a79bb77114833840f31896f9a97cc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=201, earliestPutTs=1732093206212 2024-11-20T09:00:07,806 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.Compactor(225): Compacting bc898d3b742c42e79dc1af0abba0316c, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1732093205862 2024-11-20T09:00:07,807 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.Compactor(225): Compacting be529b6eb2b54b3eae6d8c04d74ab6b1, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=201, earliestPutTs=1732093206212 2024-11-20T09:00:07,826 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f0e725d6c797c04eb86dfc3be87ab3ba#B#compaction#165 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T09:00:07,826 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/B/6b8e2b4645b4449095bac273d09947bd is 50, key is test_row_0/B:col10/1732093206860/Put/seqid=0 2024-11-20T09:00:07,828 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-20T09:00:07,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. 2024-11-20T09:00:07,828 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2902): Flushing f0e725d6c797c04eb86dfc3be87ab3ba 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-20T09:00:07,844 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=f0e725d6c797c04eb86dfc3be87ab3ba] 2024-11-20T09:00:07,848 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241120381a9f12e6b24454b0954f36b20203bb_f0e725d6c797c04eb86dfc3be87ab3ba store=[table=TestAcidGuarantees family=A region=f0e725d6c797c04eb86dfc3be87ab3ba] 2024-11-20T09:00:07,849 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742017_1193 (size=12595) 2024-11-20T09:00:07,852 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241120381a9f12e6b24454b0954f36b20203bb_f0e725d6c797c04eb86dfc3be87ab3ba, store=[table=TestAcidGuarantees family=A region=f0e725d6c797c04eb86dfc3be87ab3ba] 2024-11-20T09:00:07,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120a8fcf24fea8147a299cfedafc2529b6d_f0e725d6c797c04eb86dfc3be87ab3ba is 50, key is test_row_0/A:col10/1732093206871/Put/seqid=0 2024-11-20T09:00:07,855 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120381a9f12e6b24454b0954f36b20203bb_f0e725d6c797c04eb86dfc3be87ab3ba because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=f0e725d6c797c04eb86dfc3be87ab3ba] 2024-11-20T09:00:07,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-20T09:00:07,859 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/B/6b8e2b4645b4449095bac273d09947bd as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/B/6b8e2b4645b4449095bac273d09947bd 2024-11-20T09:00:07,868 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f0e725d6c797c04eb86dfc3be87ab3ba/B of f0e725d6c797c04eb86dfc3be87ab3ba into 6b8e2b4645b4449095bac273d09947bd(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T09:00:07,872 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f0e725d6c797c04eb86dfc3be87ab3ba: 2024-11-20T09:00:07,872 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., storeName=f0e725d6c797c04eb86dfc3be87ab3ba/B, priority=13, startTime=1732093207802; duration=0sec 2024-11-20T09:00:07,872 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T09:00:07,872 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f0e725d6c797c04eb86dfc3be87ab3ba:B 2024-11-20T09:00:07,872 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T09:00:07,874 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48828 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T09:00:07,874 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1541): f0e725d6c797c04eb86dfc3be87ab3ba/C is initiating minor compaction (all files) 2024-11-20T09:00:07,874 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f0e725d6c797c04eb86dfc3be87ab3ba/C in TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. 2024-11-20T09:00:07,874 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/C/7cad10a70d7b402381ce9c9baccd6e65, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/C/1190d219706e44279cb3f77e116de28b, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/C/b5ea089291a74d1ba31b13d0ce3c7b92, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/C/bcf5b04ce6fc4895a880cb7852a4a3dc] into tmpdir=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp, totalSize=47.7 K 2024-11-20T09:00:07,875 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742019_1195 (size=4469) 2024-11-20T09:00:07,875 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting 7cad10a70d7b402381ce9c9baccd6e65, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1732093204858 2024-11-20T09:00:07,880 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting 1190d219706e44279cb3f77e116de28b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=160, earliestPutTs=1732093205726 2024-11-20T09:00:07,880 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742018_1194 (size=12304) 2024-11-20T09:00:07,882 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f0e725d6c797c04eb86dfc3be87ab3ba#A#compaction#166 average throughput is 0.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T09:00:07,882 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting b5ea089291a74d1ba31b13d0ce3c7b92, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1732093205862 2024-11-20T09:00:07,882 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/A/9891fc6b68b640c7b419f1ea7c44812b is 175, key is test_row_0/A:col10/1732093206860/Put/seqid=0 2024-11-20T09:00:07,883 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting bcf5b04ce6fc4895a880cb7852a4a3dc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=201, earliestPutTs=1732093206212 2024-11-20T09:00:07,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:07,888 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742020_1196 (size=31515) 2024-11-20T09:00:07,890 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120a8fcf24fea8147a299cfedafc2529b6d_f0e725d6c797c04eb86dfc3be87ab3ba to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120a8fcf24fea8147a299cfedafc2529b6d_f0e725d6c797c04eb86dfc3be87ab3ba 2024-11-20T09:00:07,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/A/463417a27d77434b9257328958bf6bd6, store: [table=TestAcidGuarantees family=A region=f0e725d6c797c04eb86dfc3be87ab3ba] 2024-11-20T09:00:07,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/A/463417a27d77434b9257328958bf6bd6 is 175, key is test_row_0/A:col10/1732093206871/Put/seqid=0 2024-11-20T09:00:07,903 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/A/9891fc6b68b640c7b419f1ea7c44812b as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/A/9891fc6b68b640c7b419f1ea7c44812b 2024-11-20T09:00:07,915 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in f0e725d6c797c04eb86dfc3be87ab3ba/A of f0e725d6c797c04eb86dfc3be87ab3ba into 9891fc6b68b640c7b419f1ea7c44812b(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T09:00:07,915 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f0e725d6c797c04eb86dfc3be87ab3ba: 2024-11-20T09:00:07,915 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., storeName=f0e725d6c797c04eb86dfc3be87ab3ba/A, priority=11, startTime=1732093207801; duration=0sec 2024-11-20T09:00:07,915 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T09:00:07,915 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f0e725d6c797c04eb86dfc3be87ab3ba:A 2024-11-20T09:00:07,926 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f0e725d6c797c04eb86dfc3be87ab3ba#C#compaction#168 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T09:00:07,926 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/C/e74622048252400ebdaf91b56348ac98 is 50, key is test_row_0/C:col10/1732093206860/Put/seqid=0 2024-11-20T09:00:07,930 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742021_1197 (size=31105) 2024-11-20T09:00:07,932 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=212, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/A/463417a27d77434b9257328958bf6bd6 2024-11-20T09:00:07,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/B/28ceb0b2faf44e649b20b9718d217270 is 50, key is test_row_0/B:col10/1732093206871/Put/seqid=0 2024-11-20T09:00:07,982 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742022_1198 (size=12561) 2024-11-20T09:00:07,989 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742023_1199 (size=12151) 2024-11-20T09:00:07,991 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=212 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/B/28ceb0b2faf44e649b20b9718d217270 2024-11-20T09:00:07,995 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/C/e74622048252400ebdaf91b56348ac98 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/C/e74622048252400ebdaf91b56348ac98 2024-11-20T09:00:08,008 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f0e725d6c797c04eb86dfc3be87ab3ba/C of f0e725d6c797c04eb86dfc3be87ab3ba into e74622048252400ebdaf91b56348ac98(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T09:00:08,008 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f0e725d6c797c04eb86dfc3be87ab3ba: 2024-11-20T09:00:08,009 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., storeName=f0e725d6c797c04eb86dfc3be87ab3ba/C, priority=12, startTime=1732093207805; duration=0sec 2024-11-20T09:00:08,009 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T09:00:08,009 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f0e725d6c797c04eb86dfc3be87ab3ba:C 2024-11-20T09:00:08,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/C/4aaf4296b433406c992433b14fb73d34 is 50, key is test_row_0/C:col10/1732093206871/Put/seqid=0 2024-11-20T09:00:08,033 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742024_1200 (size=12151) 2024-11-20T09:00:08,036 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=212 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/C/4aaf4296b433406c992433b14fb73d34 2024-11-20T09:00:08,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/A/463417a27d77434b9257328958bf6bd6 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/A/463417a27d77434b9257328958bf6bd6 2024-11-20T09:00:08,049 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/A/463417a27d77434b9257328958bf6bd6, entries=150, sequenceid=212, filesize=30.4 K 2024-11-20T09:00:08,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(8855): Flush requested on f0e725d6c797c04eb86dfc3be87ab3ba 2024-11-20T09:00:08,050 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. as already flushing 2024-11-20T09:00:08,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/B/28ceb0b2faf44e649b20b9718d217270 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/B/28ceb0b2faf44e649b20b9718d217270 2024-11-20T09:00:08,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:08,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:08,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:08,057 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/B/28ceb0b2faf44e649b20b9718d217270, entries=150, sequenceid=212, filesize=11.9 K 2024-11-20T09:00:08,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:08,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:08,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:08,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/C/4aaf4296b433406c992433b14fb73d34 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/C/4aaf4296b433406c992433b14fb73d34 2024-11-20T09:00:08,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:08,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:08,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:08,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:08,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:08,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:08,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:08,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:08,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:08,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:08,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:08,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:08,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:08,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:08,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:08,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:08,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:08,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:08,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:08,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:08,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:08,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:08,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:08,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:08,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:08,066 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/C/4aaf4296b433406c992433b14fb73d34, entries=150, sequenceid=212, filesize=11.9 K 2024-11-20T09:00:08,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:08,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:08,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:08,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:08,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:08,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:08,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:08,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:08,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:08,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:08,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:08,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:08,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:08,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:08,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:08,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:08,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:08,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:08,071 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=13.42 KB/13740 for f0e725d6c797c04eb86dfc3be87ab3ba in 243ms, sequenceid=212, compaction requested=false 2024-11-20T09:00:08,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:08,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:08,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2603): Flush status journal for f0e725d6c797c04eb86dfc3be87ab3ba: 2024-11-20T09:00:08,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. 2024-11-20T09:00:08,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=41 2024-11-20T09:00:08,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:08,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:08,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.HMaster(4169): Remote procedure done, pid=41 2024-11-20T09:00:08,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:08,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:08,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:08,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:08,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:08,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:08,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:08,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:08,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:08,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:08,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:08,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:08,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:08,076 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=40 2024-11-20T09:00:08,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:08,076 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3530 sec 2024-11-20T09:00:08,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:08,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:08,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:08,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:08,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:08,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:08,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:08,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:08,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:08,079 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees in 1.3600 sec 2024-11-20T09:00:08,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:08,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:08,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:08,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:08,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:08,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:08,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:08,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:08,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:08,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:08,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:08,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:08,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:08,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:08,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:08,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:08,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:08,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:08,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:08,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:08,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:08,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:08,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:08,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:08,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:08,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:08,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:08,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:08,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:08,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:08,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:08,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:08,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:08,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:08,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:08,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:08,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:08,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:08,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:08,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:08,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:08,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:08,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:08,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:08,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:08,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:08,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:08,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:08,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:08,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:08,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:08,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:08,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:08,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:08,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:08,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:08,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:08,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:08,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:08,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:08,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:08,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:08,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:08,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:08,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:08,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:08,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:08,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:08,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:08,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:08,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:08,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:08,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:08,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:08,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:08,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:08,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:08,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:08,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:08,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:08,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:08,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:08,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:08,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:08,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:08,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:08,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:08,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:08,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:08,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:08,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:08,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:08,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:08,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(8855): Flush requested on f0e725d6c797c04eb86dfc3be87ab3ba 2024-11-20T09:00:08,117 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T09:00:08,117 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f0e725d6c797c04eb86dfc3be87ab3ba 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-20T09:00:08,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:08,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:08,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:08,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:08,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:08,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:08,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:08,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:08,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:08,128 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120ee026e55f2134e468f29a582466bda6b_f0e725d6c797c04eb86dfc3be87ab3ba is 50, key is test_row_0/A:col10/1732093208108/Put/seqid=0 2024-11-20T09:00:08,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:08,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:08,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:08,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:08,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:08,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:08,140 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742026_1202 (size=24758) 2024-11-20T09:00:08,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:08,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:08,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:08,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:08,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:08,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:08,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:08,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:08,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:08,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:08,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:08,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:08,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:08,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:08,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:08,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:08,186 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:08,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48466 deadline: 1732093268177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:08,187 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:08,187 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:08,187 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:08,187 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:08,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48394 deadline: 1732093268179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:08,188 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:08,188 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:08,188 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:08,188 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:08,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48388 deadline: 1732093268179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:08,189 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:08,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48450 deadline: 1732093268182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:08,190 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:08,190 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:08,190 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:08,190 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:08,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48420 deadline: 1732093268186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:08,190 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:08,190 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:08,190 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:08,190 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:08,190 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:08,190 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:08,303 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:08,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48466 deadline: 1732093268296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:08,304 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:08,304 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:08,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48394 deadline: 1732093268297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:08,304 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:08,304 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:08,304 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:08,304 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:08,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48388 deadline: 1732093268297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:08,304 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:08,305 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:08,305 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:08,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48450 deadline: 1732093268297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:08,305 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:08,305 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:08,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48420 deadline: 1732093268297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:08,305 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:08,305 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:08,306 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:08,306 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:08,306 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:08,306 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:08,306 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:08,306 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:08,509 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:08,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48466 deadline: 1732093268507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:08,510 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:08,510 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:08,510 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:08,520 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:08,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48394 deadline: 1732093268517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:08,521 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:08,521 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:08,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48388 deadline: 1732093268517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:08,521 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:08,521 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:08,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48420 deadline: 1732093268517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:08,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48450 deadline: 1732093268517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:08,521 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:08,521 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:08,522 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:08,522 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:08,522 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:08,522 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:08,522 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:08,522 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:08,522 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:08,522 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:08,522 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:08,541 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:08,546 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120ee026e55f2134e468f29a582466bda6b_f0e725d6c797c04eb86dfc3be87ab3ba to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120ee026e55f2134e468f29a582466bda6b_f0e725d6c797c04eb86dfc3be87ab3ba 2024-11-20T09:00:08,547 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/A/9b551bc9cc5c47468c85d935334f342e, store: [table=TestAcidGuarantees family=A region=f0e725d6c797c04eb86dfc3be87ab3ba] 2024-11-20T09:00:08,548 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/A/9b551bc9cc5c47468c85d935334f342e is 175, key is test_row_0/A:col10/1732093208108/Put/seqid=0 2024-11-20T09:00:08,551 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742025_1201 (size=74395) 2024-11-20T09:00:08,819 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:08,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48466 deadline: 1732093268817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:08,820 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:08,820 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:08,820 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:08,831 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:08,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48394 deadline: 1732093268827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:08,831 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:08,832 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:08,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48420 deadline: 1732093268827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:08,832 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:08,832 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:08,832 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:08,832 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:08,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48450 deadline: 1732093268827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:08,832 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:08,832 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:08,832 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:08,832 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:08,832 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:08,832 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:08,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48388 deadline: 1732093268827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:08,833 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:08,833 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:08,833 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:08,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-20T09:00:08,867 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T09:00:08,868 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T09:00:08,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] procedure2.ProcedureExecutor(1139): Stored pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees 2024-11-20T09:00:08,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-20T09:00:08,870 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T09:00:08,870 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T09:00:08,871 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T09:00:08,951 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=226, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/A/9b551bc9cc5c47468c85d935334f342e 2024-11-20T09:00:08,961 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/B/03fc26e057ef4b97b2d6ac238c33de40 is 50, key is test_row_0/B:col10/1732093208108/Put/seqid=0 2024-11-20T09:00:08,968 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742027_1203 (size=12151) 2024-11-20T09:00:08,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-20T09:00:09,023 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-20T09:00:09,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. 2024-11-20T09:00:09,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. as already flushing 2024-11-20T09:00:09,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. 2024-11-20T09:00:09,024 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => f0e725d6c797c04eb86dfc3be87ab3ba, NAME => 'TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:09,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => f0e725d6c797c04eb86dfc3be87ab3ba, NAME => 'TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:09,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0e725d6c797c04eb86dfc3be87ab3ba, NAME => 'TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0e725d6c797c04eb86dfc3be87ab3ba, NAME => 'TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:09,177 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-20T09:00:09,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. 2024-11-20T09:00:09,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. as already flushing 2024-11-20T09:00:09,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. 2024-11-20T09:00:09,177 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => f0e725d6c797c04eb86dfc3be87ab3ba, NAME => 'TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:09,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => f0e725d6c797c04eb86dfc3be87ab3ba, NAME => 'TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:09,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0e725d6c797c04eb86dfc3be87ab3ba, NAME => 'TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0e725d6c797c04eb86dfc3be87ab3ba, NAME => 'TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:09,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-20T09:00:09,330 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:09,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48466 deadline: 1732093269327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:09,330 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:09,330 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:09,330 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:09,331 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-20T09:00:09,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. 2024-11-20T09:00:09,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. as already flushing 2024-11-20T09:00:09,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. 2024-11-20T09:00:09,331 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => f0e725d6c797c04eb86dfc3be87ab3ba, NAME => 'TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:09,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => f0e725d6c797c04eb86dfc3be87ab3ba, NAME => 'TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:09,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0e725d6c797c04eb86dfc3be87ab3ba, NAME => 'TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0e725d6c797c04eb86dfc3be87ab3ba, NAME => 'TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:09,339 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:09,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48450 deadline: 1732093269337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:09,339 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:09,339 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:09,339 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:09,339 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:09,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48388 deadline: 1732093269337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:09,340 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:09,340 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:09,340 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:09,347 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:09,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48394 deadline: 1732093269347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:09,347 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:09,348 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:09,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48420 deadline: 1732093269347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:09,348 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:09,348 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:09,348 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:09,348 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:09,348 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:09,369 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=226 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/B/03fc26e057ef4b97b2d6ac238c33de40 2024-11-20T09:00:09,379 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/C/d1258e24721f4ea28fa6866bd79c43c2 is 50, key is test_row_0/C:col10/1732093208108/Put/seqid=0 2024-11-20T09:00:09,387 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742028_1204 (size=12151) 2024-11-20T09:00:09,484 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-20T09:00:09,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. 2024-11-20T09:00:09,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. as already flushing 2024-11-20T09:00:09,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. 2024-11-20T09:00:09,485 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => f0e725d6c797c04eb86dfc3be87ab3ba, NAME => 'TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:09,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => f0e725d6c797c04eb86dfc3be87ab3ba, NAME => 'TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:09,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0e725d6c797c04eb86dfc3be87ab3ba, NAME => 'TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0e725d6c797c04eb86dfc3be87ab3ba, NAME => 'TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:09,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-20T09:00:09,637 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-20T09:00:09,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. 2024-11-20T09:00:09,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. as already flushing 2024-11-20T09:00:09,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. 2024-11-20T09:00:09,638 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => f0e725d6c797c04eb86dfc3be87ab3ba, NAME => 'TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:09,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => f0e725d6c797c04eb86dfc3be87ab3ba, NAME => 'TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:09,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0e725d6c797c04eb86dfc3be87ab3ba, NAME => 'TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0e725d6c797c04eb86dfc3be87ab3ba, NAME => 'TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:09,788 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=226 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/C/d1258e24721f4ea28fa6866bd79c43c2 2024-11-20T09:00:09,790 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-20T09:00:09,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. 2024-11-20T09:00:09,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. as already flushing 2024-11-20T09:00:09,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. 2024-11-20T09:00:09,791 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => f0e725d6c797c04eb86dfc3be87ab3ba, NAME => 'TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:09,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => f0e725d6c797c04eb86dfc3be87ab3ba, NAME => 'TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:09,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0e725d6c797c04eb86dfc3be87ab3ba, NAME => 'TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0e725d6c797c04eb86dfc3be87ab3ba, NAME => 'TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:09,795 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/A/9b551bc9cc5c47468c85d935334f342e as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/A/9b551bc9cc5c47468c85d935334f342e 2024-11-20T09:00:09,800 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/A/9b551bc9cc5c47468c85d935334f342e, entries=400, sequenceid=226, filesize=72.7 K 2024-11-20T09:00:09,802 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/B/03fc26e057ef4b97b2d6ac238c33de40 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/B/03fc26e057ef4b97b2d6ac238c33de40 2024-11-20T09:00:09,807 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/B/03fc26e057ef4b97b2d6ac238c33de40, entries=150, sequenceid=226, filesize=11.9 K 2024-11-20T09:00:09,808 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/C/d1258e24721f4ea28fa6866bd79c43c2 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/C/d1258e24721f4ea28fa6866bd79c43c2 2024-11-20T09:00:09,823 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/C/d1258e24721f4ea28fa6866bd79c43c2, entries=150, sequenceid=226, filesize=11.9 K 2024-11-20T09:00:09,824 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for f0e725d6c797c04eb86dfc3be87ab3ba in 1707ms, sequenceid=226, compaction requested=true 2024-11-20T09:00:09,825 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f0e725d6c797c04eb86dfc3be87ab3ba: 2024-11-20T09:00:09,825 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f0e725d6c797c04eb86dfc3be87ab3ba:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T09:00:09,825 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T09:00:09,825 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T09:00:09,825 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T09:00:09,825 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f0e725d6c797c04eb86dfc3be87ab3ba:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T09:00:09,825 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T09:00:09,825 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f0e725d6c797c04eb86dfc3be87ab3ba:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T09:00:09,825 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T09:00:09,826 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T09:00:09,827 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1541): f0e725d6c797c04eb86dfc3be87ab3ba/B is initiating minor compaction (all files) 2024-11-20T09:00:09,827 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f0e725d6c797c04eb86dfc3be87ab3ba/B in TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. 2024-11-20T09:00:09,827 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/B/6b8e2b4645b4449095bac273d09947bd, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/B/28ceb0b2faf44e649b20b9718d217270, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/B/03fc26e057ef4b97b2d6ac238c33de40] into tmpdir=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp, totalSize=36.0 K 2024-11-20T09:00:09,827 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 137015 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T09:00:09,827 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HStore(1541): f0e725d6c797c04eb86dfc3be87ab3ba/A is initiating minor compaction (all files) 2024-11-20T09:00:09,827 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f0e725d6c797c04eb86dfc3be87ab3ba/A in TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. 2024-11-20T09:00:09,827 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/A/9891fc6b68b640c7b419f1ea7c44812b, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/A/463417a27d77434b9257328958bf6bd6, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/A/9b551bc9cc5c47468c85d935334f342e] into tmpdir=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp, totalSize=133.8 K 2024-11-20T09:00:09,827 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. 2024-11-20T09:00:09,827 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. files: [hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/A/9891fc6b68b640c7b419f1ea7c44812b, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/A/463417a27d77434b9257328958bf6bd6, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/A/9b551bc9cc5c47468c85d935334f342e] 2024-11-20T09:00:09,828 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting 6b8e2b4645b4449095bac273d09947bd, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=201, earliestPutTs=1732093206212 2024-11-20T09:00:09,828 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9891fc6b68b640c7b419f1ea7c44812b, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=201, earliestPutTs=1732093206212 2024-11-20T09:00:09,829 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting 28ceb0b2faf44e649b20b9718d217270, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1732093206864 2024-11-20T09:00:09,829 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.Compactor(225): Compacting 463417a27d77434b9257328958bf6bd6, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1732093206864 2024-11-20T09:00:09,829 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting 03fc26e057ef4b97b2d6ac238c33de40, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1732093208108 2024-11-20T09:00:09,829 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9b551bc9cc5c47468c85d935334f342e, keycount=400, bloomtype=ROW, size=72.7 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1732093208048 2024-11-20T09:00:09,840 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=f0e725d6c797c04eb86dfc3be87ab3ba] 2024-11-20T09:00:09,841 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f0e725d6c797c04eb86dfc3be87ab3ba#B#compaction#174 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-20T09:00:09,842 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/B/ebde470d64dc473ba2d4d3aa25cdb672 is 50, key is test_row_0/B:col10/1732093208108/Put/seqid=0 2024-11-20T09:00:09,843 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241120a444a1bcbfb84f3abbb0a4544219b16a_f0e725d6c797c04eb86dfc3be87ab3ba store=[table=TestAcidGuarantees family=A region=f0e725d6c797c04eb86dfc3be87ab3ba] 2024-11-20T09:00:09,848 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241120a444a1bcbfb84f3abbb0a4544219b16a_f0e725d6c797c04eb86dfc3be87ab3ba, store=[table=TestAcidGuarantees family=A region=f0e725d6c797c04eb86dfc3be87ab3ba] 2024-11-20T09:00:09,848 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742029_1205 (size=12697) 2024-11-20T09:00:09,848 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120a444a1bcbfb84f3abbb0a4544219b16a_f0e725d6c797c04eb86dfc3be87ab3ba because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=f0e725d6c797c04eb86dfc3be87ab3ba] 2024-11-20T09:00:09,857 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742030_1206 (size=4469) 2024-11-20T09:00:09,858 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/B/ebde470d64dc473ba2d4d3aa25cdb672 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/B/ebde470d64dc473ba2d4d3aa25cdb672 2024-11-20T09:00:09,859 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f0e725d6c797c04eb86dfc3be87ab3ba#A#compaction#175 average throughput is 1.36 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T09:00:09,859 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/A/a598c6511e204fa2872c9a6bfc8e04b4 is 175, key is test_row_0/A:col10/1732093208108/Put/seqid=0 2024-11-20T09:00:09,865 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f0e725d6c797c04eb86dfc3be87ab3ba/B of f0e725d6c797c04eb86dfc3be87ab3ba into ebde470d64dc473ba2d4d3aa25cdb672(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T09:00:09,865 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f0e725d6c797c04eb86dfc3be87ab3ba: 2024-11-20T09:00:09,865 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., storeName=f0e725d6c797c04eb86dfc3be87ab3ba/B, priority=13, startTime=1732093209825; duration=0sec 2024-11-20T09:00:09,865 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T09:00:09,865 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f0e725d6c797c04eb86dfc3be87ab3ba:B 2024-11-20T09:00:09,865 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T09:00:09,866 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T09:00:09,866 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1541): f0e725d6c797c04eb86dfc3be87ab3ba/C is initiating minor compaction (all files) 2024-11-20T09:00:09,866 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f0e725d6c797c04eb86dfc3be87ab3ba/C in TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. 2024-11-20T09:00:09,866 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/C/e74622048252400ebdaf91b56348ac98, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/C/4aaf4296b433406c992433b14fb73d34, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/C/d1258e24721f4ea28fa6866bd79c43c2] into tmpdir=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp, totalSize=36.0 K 2024-11-20T09:00:09,867 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting e74622048252400ebdaf91b56348ac98, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=201, earliestPutTs=1732093206212 2024-11-20T09:00:09,867 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting 4aaf4296b433406c992433b14fb73d34, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1732093206864 2024-11-20T09:00:09,868 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting d1258e24721f4ea28fa6866bd79c43c2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1732093208108 2024-11-20T09:00:09,877 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742031_1207 (size=31617) 2024-11-20T09:00:09,879 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f0e725d6c797c04eb86dfc3be87ab3ba#C#compaction#176 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T09:00:09,880 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/C/1f55295c3ec0486cb108bddf3343dcbc is 50, key is test_row_0/C:col10/1732093208108/Put/seqid=0 2024-11-20T09:00:09,887 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/A/a598c6511e204fa2872c9a6bfc8e04b4 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/A/a598c6511e204fa2872c9a6bfc8e04b4 2024-11-20T09:00:09,894 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f0e725d6c797c04eb86dfc3be87ab3ba/A of f0e725d6c797c04eb86dfc3be87ab3ba into a598c6511e204fa2872c9a6bfc8e04b4(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T09:00:09,894 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f0e725d6c797c04eb86dfc3be87ab3ba: 2024-11-20T09:00:09,894 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., storeName=f0e725d6c797c04eb86dfc3be87ab3ba/A, priority=13, startTime=1732093209825; duration=0sec 2024-11-20T09:00:09,894 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T09:00:09,894 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f0e725d6c797c04eb86dfc3be87ab3ba:A 2024-11-20T09:00:09,915 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742032_1208 (size=12663) 2024-11-20T09:00:09,922 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/C/1f55295c3ec0486cb108bddf3343dcbc as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/C/1f55295c3ec0486cb108bddf3343dcbc 2024-11-20T09:00:09,930 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f0e725d6c797c04eb86dfc3be87ab3ba/C of f0e725d6c797c04eb86dfc3be87ab3ba into 1f55295c3ec0486cb108bddf3343dcbc(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T09:00:09,931 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f0e725d6c797c04eb86dfc3be87ab3ba: 2024-11-20T09:00:09,931 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., storeName=f0e725d6c797c04eb86dfc3be87ab3ba/C, priority=13, startTime=1732093209825; duration=0sec 2024-11-20T09:00:09,931 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T09:00:09,931 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f0e725d6c797c04eb86dfc3be87ab3ba:C 2024-11-20T09:00:09,945 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-20T09:00:09,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. 2024-11-20T09:00:09,945 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2902): Flushing f0e725d6c797c04eb86dfc3be87ab3ba 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-20T09:00:09,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411205d3df700967f4b91bc0f6bd9ac1314cd_f0e725d6c797c04eb86dfc3be87ab3ba is 50, key is test_row_0/A:col10/1732093208170/Put/seqid=0 2024-11-20T09:00:09,959 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742033_1209 (size=12304) 2024-11-20T09:00:10,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-20T09:00:10,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(8855): Flush requested on f0e725d6c797c04eb86dfc3be87ab3ba 2024-11-20T09:00:10,339 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. as already flushing 2024-11-20T09:00:10,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:10,363 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:10,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48466 deadline: 1732093270362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:10,365 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:10,365 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:10,365 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:10,368 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411205d3df700967f4b91bc0f6bd9ac1314cd_f0e725d6c797c04eb86dfc3be87ab3ba to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411205d3df700967f4b91bc0f6bd9ac1314cd_f0e725d6c797c04eb86dfc3be87ab3ba 2024-11-20T09:00:10,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/A/b8da9a8c4c50477d91f8dddebc8346d9, store: [table=TestAcidGuarantees family=A region=f0e725d6c797c04eb86dfc3be87ab3ba] 2024-11-20T09:00:10,370 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:10,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48450 deadline: 1732093270364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:10,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/A/b8da9a8c4c50477d91f8dddebc8346d9 is 175, key is test_row_0/A:col10/1732093208170/Put/seqid=0 2024-11-20T09:00:10,371 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:10,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48388 deadline: 1732093270365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:10,371 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:10,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48394 deadline: 1732093270367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:10,372 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:10,372 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:10,372 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:10,372 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:10,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48420 deadline: 1732093270367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:10,372 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to be1eb2620e0e,34223,1732093159960 for 'test_row_1' in f0e725d6c797c04eb86dfc3be87ab3ba of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2186 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-20T09:00:10,373 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:10,373 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:10,373 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:10,373 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to be1eb2620e0e,34223,1732093159960 for 'test_row_1' in f0e725d6c797c04eb86dfc3be87ab3ba of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2194 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-20T09:00:10,374 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:10,374 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:10,374 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:10,375 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:10,375 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:10,375 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:10,391 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742034_1210 (size=31105) 2024-11-20T09:00:10,393 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=254, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/A/b8da9a8c4c50477d91f8dddebc8346d9 2024-11-20T09:00:10,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/B/cb06fa40ddbe4f9d8f4ad4982c6c01b3 is 50, key is test_row_0/B:col10/1732093208170/Put/seqid=0 2024-11-20T09:00:10,428 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742035_1211 (size=12151) 2024-11-20T09:00:10,477 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:10,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48466 deadline: 1732093270477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:10,478 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:10,478 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:10,478 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:10,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48450 deadline: 1732093270477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:10,478 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:10,478 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:10,478 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:10,478 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:10,490 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:10,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48388 deadline: 1732093270487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:10,491 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:10,491 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:10,491 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:10,692 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:10,692 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:10,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48466 deadline: 1732093270687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:10,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48450 deadline: 1732093270687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:10,693 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:10,693 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:10,693 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:10,693 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:10,693 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:10,693 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:10,698 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:10,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48388 deadline: 1732093270697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:10,698 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:10,698 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:10,698 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:10,829 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=254 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/B/cb06fa40ddbe4f9d8f4ad4982c6c01b3 2024-11-20T09:00:10,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/C/5a50a19faa154f1eabb1df95b5931a65 is 50, key is test_row_0/C:col10/1732093208170/Put/seqid=0 2024-11-20T09:00:10,842 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742036_1212 (size=12151) 2024-11-20T09:00:11,000 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:11,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48466 deadline: 1732093270997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:11,000 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:11,000 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:11,000 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:11,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48450 deadline: 1732093270997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:11,000 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:11,001 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:11,001 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:11,001 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:11,010 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:11,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48388 deadline: 1732093271007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:11,010 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:11,010 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:11,010 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:11,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-20T09:00:11,243 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=254 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/C/5a50a19faa154f1eabb1df95b5931a65 2024-11-20T09:00:11,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/A/b8da9a8c4c50477d91f8dddebc8346d9 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/A/b8da9a8c4c50477d91f8dddebc8346d9 2024-11-20T09:00:11,256 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/A/b8da9a8c4c50477d91f8dddebc8346d9, entries=150, sequenceid=254, filesize=30.4 K 2024-11-20T09:00:11,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/B/cb06fa40ddbe4f9d8f4ad4982c6c01b3 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/B/cb06fa40ddbe4f9d8f4ad4982c6c01b3 2024-11-20T09:00:11,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,262 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/B/cb06fa40ddbe4f9d8f4ad4982c6c01b3, entries=150, sequenceid=254, filesize=11.9 K 2024-11-20T09:00:11,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/C/5a50a19faa154f1eabb1df95b5931a65 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/C/5a50a19faa154f1eabb1df95b5931a65 2024-11-20T09:00:11,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,269 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/C/5a50a19faa154f1eabb1df95b5931a65, entries=150, sequenceid=254, filesize=11.9 K 2024-11-20T09:00:11,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,271 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=60.38 KB/61830 for f0e725d6c797c04eb86dfc3be87ab3ba in 1326ms, sequenceid=254, compaction requested=false 2024-11-20T09:00:11,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2603): Flush status journal for f0e725d6c797c04eb86dfc3be87ab3ba: 2024-11-20T09:00:11,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. 2024-11-20T09:00:11,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=43 2024-11-20T09:00:11,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.HMaster(4169): Remote procedure done, pid=43 2024-11-20T09:00:11,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,275 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=42 2024-11-20T09:00:11,275 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4020 sec 2024-11-20T09:00:11,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,278 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees in 2.4080 sec 2024-11-20T09:00:11,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(8855): Flush requested on f0e725d6c797c04eb86dfc3be87ab3ba 2024-11-20T09:00:11,512 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T09:00:11,512 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f0e725d6c797c04eb86dfc3be87ab3ba 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-20T09:00:11,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,531 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411209856c6312834472b8e45d3b103637f94_f0e725d6c797c04eb86dfc3be87ab3ba is 50, key is test_row_0/A:col10/1732093210362/Put/seqid=0 2024-11-20T09:00:11,553 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742037_1213 (size=12454) 2024-11-20T09:00:11,571 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:11,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48388 deadline: 1732093271563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:11,572 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:11,572 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:11,572 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:11,572 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:11,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48450 deadline: 1732093271572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:11,573 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:11,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48466 deadline: 1732093271572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:11,575 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:11,575 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:11,575 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:11,575 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:11,575 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:11,575 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:11,679 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:11,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48388 deadline: 1732093271677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:11,680 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:11,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48450 deadline: 1732093271677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:11,680 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:11,680 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:11,680 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:11,680 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:11,680 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:11,680 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:11,688 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:11,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48466 deadline: 1732093271687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:11,689 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:11,689 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:11,689 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:11,890 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:11,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48388 deadline: 1732093271887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:11,891 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:11,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48450 deadline: 1732093271887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:11,891 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:11,891 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:11,891 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:11,891 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:11,891 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:11,891 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:11,898 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:11,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48466 deadline: 1732093271897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:11,898 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:11,898 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:11,898 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:11,954 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:11,966 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411209856c6312834472b8e45d3b103637f94_f0e725d6c797c04eb86dfc3be87ab3ba to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411209856c6312834472b8e45d3b103637f94_f0e725d6c797c04eb86dfc3be87ab3ba 2024-11-20T09:00:11,970 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/A/7bddf792bd6c4490854387b9471a09fa, store: [table=TestAcidGuarantees family=A region=f0e725d6c797c04eb86dfc3be87ab3ba] 2024-11-20T09:00:11,970 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/A/7bddf792bd6c4490854387b9471a09fa is 175, key is test_row_0/A:col10/1732093210362/Put/seqid=0 2024-11-20T09:00:11,977 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742038_1214 (size=31251) 2024-11-20T09:00:11,978 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=267, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/A/7bddf792bd6c4490854387b9471a09fa 2024-11-20T09:00:11,988 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/B/e7241f8e27094568be4bd692dccd1ee8 is 50, key is test_row_0/B:col10/1732093210362/Put/seqid=0 2024-11-20T09:00:11,996 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742039_1215 (size=9857) 2024-11-20T09:00:11,997 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=267 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/B/e7241f8e27094568be4bd692dccd1ee8 2024-11-20T09:00:12,017 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/C/1a5f1d8050704d5a817a49219f255199 is 50, key is test_row_0/C:col10/1732093210362/Put/seqid=0 2024-11-20T09:00:12,021 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742040_1216 (size=9857) 2024-11-20T09:00:12,023 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=267 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/C/1a5f1d8050704d5a817a49219f255199 2024-11-20T09:00:12,036 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/A/7bddf792bd6c4490854387b9471a09fa as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/A/7bddf792bd6c4490854387b9471a09fa 2024-11-20T09:00:12,045 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/A/7bddf792bd6c4490854387b9471a09fa, entries=150, sequenceid=267, filesize=30.5 K 2024-11-20T09:00:12,048 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/B/e7241f8e27094568be4bd692dccd1ee8 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/B/e7241f8e27094568be4bd692dccd1ee8 2024-11-20T09:00:12,054 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/B/e7241f8e27094568be4bd692dccd1ee8, entries=100, sequenceid=267, filesize=9.6 K 2024-11-20T09:00:12,055 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/C/1a5f1d8050704d5a817a49219f255199 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/C/1a5f1d8050704d5a817a49219f255199 2024-11-20T09:00:12,065 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/C/1a5f1d8050704d5a817a49219f255199, entries=100, sequenceid=267, filesize=9.6 K 2024-11-20T09:00:12,066 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for f0e725d6c797c04eb86dfc3be87ab3ba in 554ms, sequenceid=267, compaction requested=true 2024-11-20T09:00:12,066 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f0e725d6c797c04eb86dfc3be87ab3ba: 2024-11-20T09:00:12,066 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f0e725d6c797c04eb86dfc3be87ab3ba:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T09:00:12,066 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T09:00:12,066 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T09:00:12,066 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f0e725d6c797c04eb86dfc3be87ab3ba:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T09:00:12,066 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T09:00:12,066 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f0e725d6c797c04eb86dfc3be87ab3ba:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T09:00:12,066 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T09:00:12,066 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T09:00:12,067 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34705 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T09:00:12,067 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1541): f0e725d6c797c04eb86dfc3be87ab3ba/B is initiating minor compaction (all files) 2024-11-20T09:00:12,068 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f0e725d6c797c04eb86dfc3be87ab3ba/B in TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. 2024-11-20T09:00:12,068 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/B/ebde470d64dc473ba2d4d3aa25cdb672, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/B/cb06fa40ddbe4f9d8f4ad4982c6c01b3, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/B/e7241f8e27094568be4bd692dccd1ee8] into tmpdir=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp, totalSize=33.9 K 2024-11-20T09:00:12,069 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93973 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T09:00:12,069 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HStore(1541): f0e725d6c797c04eb86dfc3be87ab3ba/A is initiating minor compaction (all files) 2024-11-20T09:00:12,069 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f0e725d6c797c04eb86dfc3be87ab3ba/A in TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. 2024-11-20T09:00:12,069 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/A/a598c6511e204fa2872c9a6bfc8e04b4, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/A/b8da9a8c4c50477d91f8dddebc8346d9, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/A/7bddf792bd6c4490854387b9471a09fa] into tmpdir=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp, totalSize=91.8 K 2024-11-20T09:00:12,069 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. 2024-11-20T09:00:12,069 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. files: [hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/A/a598c6511e204fa2872c9a6bfc8e04b4, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/A/b8da9a8c4c50477d91f8dddebc8346d9, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/A/7bddf792bd6c4490854387b9471a09fa] 2024-11-20T09:00:12,070 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.Compactor(225): Compacting a598c6511e204fa2872c9a6bfc8e04b4, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1732093208108 2024-11-20T09:00:12,070 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting ebde470d64dc473ba2d4d3aa25cdb672, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1732093208108 2024-11-20T09:00:12,070 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting cb06fa40ddbe4f9d8f4ad4982c6c01b3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1732093208170 2024-11-20T09:00:12,071 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.Compactor(225): Compacting b8da9a8c4c50477d91f8dddebc8346d9, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1732093208170 2024-11-20T09:00:12,072 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting e7241f8e27094568be4bd692dccd1ee8, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1732093210362 2024-11-20T09:00:12,074 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7bddf792bd6c4490854387b9471a09fa, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1732093210361 2024-11-20T09:00:12,086 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f0e725d6c797c04eb86dfc3be87ab3ba#B#compaction#183 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T09:00:12,086 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/B/7441c27cf52845cfb0214f8acfc69435 is 50, key is test_row_0/B:col10/1732093210362/Put/seqid=0 2024-11-20T09:00:12,107 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=f0e725d6c797c04eb86dfc3be87ab3ba] 2024-11-20T09:00:12,114 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241120992dcbed9b9046d4b2eab9176049c30f_f0e725d6c797c04eb86dfc3be87ab3ba store=[table=TestAcidGuarantees family=A region=f0e725d6c797c04eb86dfc3be87ab3ba] 2024-11-20T09:00:12,115 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241120992dcbed9b9046d4b2eab9176049c30f_f0e725d6c797c04eb86dfc3be87ab3ba, store=[table=TestAcidGuarantees family=A region=f0e725d6c797c04eb86dfc3be87ab3ba] 2024-11-20T09:00:12,115 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120992dcbed9b9046d4b2eab9176049c30f_f0e725d6c797c04eb86dfc3be87ab3ba because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=f0e725d6c797c04eb86dfc3be87ab3ba] 2024-11-20T09:00:12,130 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742041_1217 (size=12899) 2024-11-20T09:00:12,135 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742042_1218 (size=4469) 2024-11-20T09:00:12,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(8855): Flush requested on f0e725d6c797c04eb86dfc3be87ab3ba 2024-11-20T09:00:12,200 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T09:00:12,200 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f0e725d6c797c04eb86dfc3be87ab3ba 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-20T09:00:12,207 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112059af40569c5444eda0e0f6bf6a9a1b0b_f0e725d6c797c04eb86dfc3be87ab3ba is 50, key is test_row_0/A:col10/1732093211570/Put/seqid=0 2024-11-20T09:00:12,212 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742043_1219 (size=17534) 2024-11-20T09:00:12,216 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:12,222 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112059af40569c5444eda0e0f6bf6a9a1b0b_f0e725d6c797c04eb86dfc3be87ab3ba to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112059af40569c5444eda0e0f6bf6a9a1b0b_f0e725d6c797c04eb86dfc3be87ab3ba 2024-11-20T09:00:12,223 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/A/0e5829fa152e4526bda4d103be67f637, store: [table=TestAcidGuarantees family=A region=f0e725d6c797c04eb86dfc3be87ab3ba] 2024-11-20T09:00:12,224 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/A/0e5829fa152e4526bda4d103be67f637 is 175, key is test_row_0/A:col10/1732093211570/Put/seqid=0 2024-11-20T09:00:12,229 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742044_1220 (size=48639) 2024-11-20T09:00:12,257 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:12,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48388 deadline: 1732093272252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:12,257 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:12,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48466 deadline: 1732093272253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:12,258 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:12,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48450 deadline: 1732093272257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:12,258 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:12,258 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:12,258 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:12,259 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:12,259 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:12,259 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:12,259 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:12,259 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:12,259 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:12,367 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:12,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48466 deadline: 1732093272367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:12,368 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:12,368 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:12,368 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:12,369 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:12,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48450 deadline: 1732093272367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:12,369 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:12,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48388 deadline: 1732093272367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:12,369 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:12,370 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:12,370 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:12,370 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:12,370 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:12,370 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:12,389 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:12,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48420 deadline: 1732093272387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:12,389 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to be1eb2620e0e,34223,1732093159960 for 'test_row_1' in f0e725d6c797c04eb86dfc3be87ab3ba of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4203 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-20T09:00:12,390 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:12,390 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:12,390 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:12,397 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:12,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48394 deadline: 1732093272396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:12,397 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to be1eb2620e0e,34223,1732093159960 for 'test_row_1' in f0e725d6c797c04eb86dfc3be87ab3ba of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4218 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-20T09:00:12,398 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:12,398 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:12,398 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:12,537 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f0e725d6c797c04eb86dfc3be87ab3ba#A#compaction#184 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T09:00:12,537 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/A/755ccc014a9e4042855401cc641be14f is 175, key is test_row_0/A:col10/1732093210362/Put/seqid=0 2024-11-20T09:00:12,539 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/B/7441c27cf52845cfb0214f8acfc69435 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/B/7441c27cf52845cfb0214f8acfc69435 2024-11-20T09:00:12,546 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f0e725d6c797c04eb86dfc3be87ab3ba/B of f0e725d6c797c04eb86dfc3be87ab3ba into 7441c27cf52845cfb0214f8acfc69435(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T09:00:12,546 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f0e725d6c797c04eb86dfc3be87ab3ba: 2024-11-20T09:00:12,546 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., storeName=f0e725d6c797c04eb86dfc3be87ab3ba/B, priority=13, startTime=1732093212066; duration=0sec 2024-11-20T09:00:12,546 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T09:00:12,546 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f0e725d6c797c04eb86dfc3be87ab3ba:B 2024-11-20T09:00:12,546 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T09:00:12,547 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34671 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T09:00:12,548 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1541): f0e725d6c797c04eb86dfc3be87ab3ba/C is initiating minor compaction (all files) 2024-11-20T09:00:12,548 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f0e725d6c797c04eb86dfc3be87ab3ba/C in TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. 2024-11-20T09:00:12,548 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/C/1f55295c3ec0486cb108bddf3343dcbc, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/C/5a50a19faa154f1eabb1df95b5931a65, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/C/1a5f1d8050704d5a817a49219f255199] into tmpdir=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp, totalSize=33.9 K 2024-11-20T09:00:12,548 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting 1f55295c3ec0486cb108bddf3343dcbc, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1732093208108 2024-11-20T09:00:12,549 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting 5a50a19faa154f1eabb1df95b5931a65, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1732093208170 2024-11-20T09:00:12,549 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting 1a5f1d8050704d5a817a49219f255199, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1732093210362 2024-11-20T09:00:12,561 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742045_1221 (size=31926) 2024-11-20T09:00:12,564 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f0e725d6c797c04eb86dfc3be87ab3ba#C#compaction#186 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T09:00:12,565 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/C/27dbbdc0cebd4201ad0eb2d133f4f2e9 is 50, key is test_row_0/C:col10/1732093210362/Put/seqid=0 2024-11-20T09:00:12,570 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/A/755ccc014a9e4042855401cc641be14f as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/A/755ccc014a9e4042855401cc641be14f 2024-11-20T09:00:12,577 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f0e725d6c797c04eb86dfc3be87ab3ba/A of f0e725d6c797c04eb86dfc3be87ab3ba into 755ccc014a9e4042855401cc641be14f(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T09:00:12,578 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f0e725d6c797c04eb86dfc3be87ab3ba: 2024-11-20T09:00:12,578 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., storeName=f0e725d6c797c04eb86dfc3be87ab3ba/A, priority=13, startTime=1732093212066; duration=0sec 2024-11-20T09:00:12,578 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T09:00:12,578 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f0e725d6c797c04eb86dfc3be87ab3ba:A 2024-11-20T09:00:12,580 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:12,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48450 deadline: 1732093272577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:12,580 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:12,580 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:12,580 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:12,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48466 deadline: 1732093272577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:12,580 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:12,581 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:12,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48388 deadline: 1732093272578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:12,581 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:12,581 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:12,581 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:12,581 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:12,581 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:12,581 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:12,594 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742046_1222 (size=12865) 2024-11-20T09:00:12,601 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/C/27dbbdc0cebd4201ad0eb2d133f4f2e9 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/C/27dbbdc0cebd4201ad0eb2d133f4f2e9 2024-11-20T09:00:12,609 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f0e725d6c797c04eb86dfc3be87ab3ba/C of f0e725d6c797c04eb86dfc3be87ab3ba into 27dbbdc0cebd4201ad0eb2d133f4f2e9(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T09:00:12,610 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f0e725d6c797c04eb86dfc3be87ab3ba: 2024-11-20T09:00:12,610 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., storeName=f0e725d6c797c04eb86dfc3be87ab3ba/C, priority=13, startTime=1732093212066; duration=0sec 2024-11-20T09:00:12,610 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T09:00:12,610 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f0e725d6c797c04eb86dfc3be87ab3ba:C 2024-11-20T09:00:12,630 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=292, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/A/0e5829fa152e4526bda4d103be67f637 2024-11-20T09:00:12,640 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/B/7757629ea3fb4086a471d6baa926888a is 50, key is test_row_0/B:col10/1732093211570/Put/seqid=0 2024-11-20T09:00:12,659 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742047_1223 (size=12301) 2024-11-20T09:00:12,889 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:12,889 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:12,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48450 deadline: 1732093272887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:12,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48466 deadline: 1732093272887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:12,889 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:12,889 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:12,889 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:12,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48388 deadline: 1732093272888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:12,890 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:12,890 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:12,890 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:12,890 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:12,890 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:12,890 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:12,890 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:13,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-20T09:00:13,027 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T09:00:13,028 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T09:00:13,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] procedure2.ProcedureExecutor(1139): Stored pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees 2024-11-20T09:00:13,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-20T09:00:13,030 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T09:00:13,031 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T09:00:13,031 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T09:00:13,060 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=292 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/B/7757629ea3fb4086a471d6baa926888a 2024-11-20T09:00:13,069 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/C/dc4c8f65ee594a12b6545f4d9667bbff is 50, key is test_row_0/C:col10/1732093211570/Put/seqid=0 2024-11-20T09:00:13,074 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742048_1224 (size=12301) 2024-11-20T09:00:13,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-20T09:00:13,184 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-20T09:00:13,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. 2024-11-20T09:00:13,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. as already flushing 2024-11-20T09:00:13,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. 2024-11-20T09:00:13,184 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => f0e725d6c797c04eb86dfc3be87ab3ba, NAME => 'TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:13,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => f0e725d6c797c04eb86dfc3be87ab3ba, NAME => 'TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:13,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0e725d6c797c04eb86dfc3be87ab3ba, NAME => 'TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0e725d6c797c04eb86dfc3be87ab3ba, NAME => 'TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:13,339 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-20T09:00:13,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. 2024-11-20T09:00:13,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. as already flushing 2024-11-20T09:00:13,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. 2024-11-20T09:00:13,340 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => f0e725d6c797c04eb86dfc3be87ab3ba, NAME => 'TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:13,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => f0e725d6c797c04eb86dfc3be87ab3ba, NAME => 'TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:13,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0e725d6c797c04eb86dfc3be87ab3ba, NAME => 'TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0e725d6c797c04eb86dfc3be87ab3ba, NAME => 'TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:13,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-20T09:00:13,399 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:13,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48466 deadline: 1732093273397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:13,400 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:13,400 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:13,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48450 deadline: 1732093273397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:13,400 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:13,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48388 deadline: 1732093273397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:13,400 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:13,400 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:13,400 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:13,400 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:13,400 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:13,400 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:13,400 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:13,400 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:13,474 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=292 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/C/dc4c8f65ee594a12b6545f4d9667bbff 2024-11-20T09:00:13,479 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/A/0e5829fa152e4526bda4d103be67f637 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/A/0e5829fa152e4526bda4d103be67f637 2024-11-20T09:00:13,485 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/A/0e5829fa152e4526bda4d103be67f637, entries=250, sequenceid=292, filesize=47.5 K 2024-11-20T09:00:13,486 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/B/7757629ea3fb4086a471d6baa926888a as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/B/7757629ea3fb4086a471d6baa926888a 2024-11-20T09:00:13,493 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-20T09:00:13,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. 2024-11-20T09:00:13,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. as already flushing 2024-11-20T09:00:13,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. 2024-11-20T09:00:13,494 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => f0e725d6c797c04eb86dfc3be87ab3ba, NAME => 'TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:13,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => f0e725d6c797c04eb86dfc3be87ab3ba, NAME => 'TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:13,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0e725d6c797c04eb86dfc3be87ab3ba, NAME => 'TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0e725d6c797c04eb86dfc3be87ab3ba, NAME => 'TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:13,497 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/B/7757629ea3fb4086a471d6baa926888a, entries=150, sequenceid=292, filesize=12.0 K 2024-11-20T09:00:13,500 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/C/dc4c8f65ee594a12b6545f4d9667bbff as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/C/dc4c8f65ee594a12b6545f4d9667bbff 2024-11-20T09:00:13,505 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/C/dc4c8f65ee594a12b6545f4d9667bbff, entries=150, sequenceid=292, filesize=12.0 K 2024-11-20T09:00:13,506 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for f0e725d6c797c04eb86dfc3be87ab3ba in 1305ms, sequenceid=292, compaction requested=false 2024-11-20T09:00:13,506 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f0e725d6c797c04eb86dfc3be87ab3ba: 2024-11-20T09:00:13,648 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-20T09:00:13,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. 2024-11-20T09:00:13,648 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2902): Flushing f0e725d6c797c04eb86dfc3be87ab3ba 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-20T09:00:13,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411209ebd9c801ef2461592da50bb6b464429_f0e725d6c797c04eb86dfc3be87ab3ba is 50, key is test_row_0/A:col10/1732093212256/Put/seqid=0 2024-11-20T09:00:13,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-20T09:00:13,660 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742049_1225 (size=12454) 2024-11-20T09:00:13,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:13,666 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411209ebd9c801ef2461592da50bb6b464429_f0e725d6c797c04eb86dfc3be87ab3ba to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411209ebd9c801ef2461592da50bb6b464429_f0e725d6c797c04eb86dfc3be87ab3ba 2024-11-20T09:00:13,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/A/e715ea8b214146aea232c62507b45252, store: [table=TestAcidGuarantees family=A region=f0e725d6c797c04eb86dfc3be87ab3ba] 2024-11-20T09:00:13,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/A/e715ea8b214146aea232c62507b45252 is 175, key is test_row_0/A:col10/1732093212256/Put/seqid=0 2024-11-20T09:00:13,671 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742050_1226 (size=31255) 2024-11-20T09:00:13,672 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=306, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/A/e715ea8b214146aea232c62507b45252 2024-11-20T09:00:13,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/B/70fc1509b6d84c69b509de5492eabb5c is 50, key is test_row_0/B:col10/1732093212256/Put/seqid=0 2024-11-20T09:00:13,684 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742051_1227 (size=12301) 2024-11-20T09:00:14,085 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=306 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/B/70fc1509b6d84c69b509de5492eabb5c 2024-11-20T09:00:14,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/C/8caf0f987d6b4226bd4f4f19cd0fdedd is 50, key is test_row_0/C:col10/1732093212256/Put/seqid=0 2024-11-20T09:00:14,134 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742052_1228 (size=12301) 2024-11-20T09:00:14,135 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=306 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/C/8caf0f987d6b4226bd4f4f19cd0fdedd 2024-11-20T09:00:14,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/A/e715ea8b214146aea232c62507b45252 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/A/e715ea8b214146aea232c62507b45252 2024-11-20T09:00:14,149 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/A/e715ea8b214146aea232c62507b45252, entries=150, sequenceid=306, filesize=30.5 K 2024-11-20T09:00:14,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/B/70fc1509b6d84c69b509de5492eabb5c as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/B/70fc1509b6d84c69b509de5492eabb5c 2024-11-20T09:00:14,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,156 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/B/70fc1509b6d84c69b509de5492eabb5c, entries=150, sequenceid=306, filesize=12.0 K 2024-11-20T09:00:14,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/C/8caf0f987d6b4226bd4f4f19cd0fdedd as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/C/8caf0f987d6b4226bd4f4f19cd0fdedd 2024-11-20T09:00:14,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,164 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/C/8caf0f987d6b4226bd4f4f19cd0fdedd, entries=150, sequenceid=306, filesize=12.0 K 2024-11-20T09:00:14,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,165 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=0 B/0 for f0e725d6c797c04eb86dfc3be87ab3ba in 517ms, sequenceid=306, compaction requested=true 2024-11-20T09:00:14,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2603): Flush status journal for f0e725d6c797c04eb86dfc3be87ab3ba: 2024-11-20T09:00:14,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. 2024-11-20T09:00:14,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=45 2024-11-20T09:00:14,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.HMaster(4169): Remote procedure done, pid=45 2024-11-20T09:00:14,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-20T09:00:14,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,168 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=44 2024-11-20T09:00:14,169 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1350 sec 2024-11-20T09:00:14,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,171 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees in 1.1410 sec 2024-11-20T09:00:14,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(8855): Flush requested on f0e725d6c797c04eb86dfc3be87ab3ba 2024-11-20T09:00:14,441 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T09:00:14,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,441 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f0e725d6c797c04eb86dfc3be87ab3ba 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-20T09:00:14,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,452 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411208820fb202895484fb7ed9f4f02ea40ff_f0e725d6c797c04eb86dfc3be87ab3ba is 50, key is test_row_0/A:col10/1732093214433/Put/seqid=0 2024-11-20T09:00:14,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742054_1230 (size=25158) 2024-11-20T09:00:14,470 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,477 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411208820fb202895484fb7ed9f4f02ea40ff_f0e725d6c797c04eb86dfc3be87ab3ba to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411208820fb202895484fb7ed9f4f02ea40ff_f0e725d6c797c04eb86dfc3be87ab3ba 2024-11-20T09:00:14,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,479 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/A/7421fef9056b4903b6a3c7c36fbacbb7, store: [table=TestAcidGuarantees family=A region=f0e725d6c797c04eb86dfc3be87ab3ba] 2024-11-20T09:00:14,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,480 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/A/7421fef9056b4903b6a3c7c36fbacbb7 is 175, key is test_row_0/A:col10/1732093214433/Put/seqid=0 2024-11-20T09:00:14,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,481 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742053_1229 (size=74795) 2024-11-20T09:00:14,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,483 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=317, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/A/7421fef9056b4903b6a3c7c36fbacbb7 2024-11-20T09:00:14,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,495 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/B/38552bf1ea2c4929b4d11bafc15b2bf7 is 50, key is test_row_0/B:col10/1732093214433/Put/seqid=0 2024-11-20T09:00:14,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,505 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742055_1231 (size=24505) 2024-11-20T09:00:14,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,510 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:14,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48388 deadline: 1732093274504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:14,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,510 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:14,511 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:14,511 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:14,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,514 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:14,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48450 deadline: 1732093274507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:14,514 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:14,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48466 deadline: 1732093274507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:14,514 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:14,514 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:14,515 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:14,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,515 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:14,515 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:14,515 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:14,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:14,622 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:14,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48388 deadline: 1732093274617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:14,623 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:14,623 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:14,623 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:14,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48466 deadline: 1732093274617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:14,623 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:14,623 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:14,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48450 deadline: 1732093274617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:14,623 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:14,623 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:14,623 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:14,623 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:14,623 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:14,623 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:14,828 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:14,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48388 deadline: 1732093274826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:14,828 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:14,828 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:14,828 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:14,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48466 deadline: 1732093274827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:14,828 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:14,829 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:14,829 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:14,829 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:14,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48450 deadline: 1732093274827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:14,829 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:14,829 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:14,829 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:14,829 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:14,906 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=317 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/B/38552bf1ea2c4929b4d11bafc15b2bf7 2024-11-20T09:00:14,914 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/C/030d34f5cdcd4e64817eb41c16b5d36c is 50, key is test_row_0/C:col10/1732093214433/Put/seqid=0 2024-11-20T09:00:14,920 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742056_1232 (size=12301) 2024-11-20T09:00:15,144 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:15,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48388 deadline: 1732093275137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:15,144 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:15,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48466 deadline: 1732093275137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:15,144 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:15,144 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:15,144 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:15,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48450 deadline: 1732093275137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:15,144 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:15,144 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:15,144 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:15,144 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:15,145 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:15,145 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:15,145 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:15,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-20T09:00:15,177 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T09:00:15,177 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T09:00:15,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] procedure2.ProcedureExecutor(1139): Stored pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees 2024-11-20T09:00:15,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-20T09:00:15,179 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T09:00:15,180 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T09:00:15,180 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T09:00:15,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-20T09:00:15,319 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=317 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/C/030d34f5cdcd4e64817eb41c16b5d36c 2024-11-20T09:00:15,324 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/A/7421fef9056b4903b6a3c7c36fbacbb7 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/A/7421fef9056b4903b6a3c7c36fbacbb7 2024-11-20T09:00:15,328 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/A/7421fef9056b4903b6a3c7c36fbacbb7, entries=400, sequenceid=317, filesize=73.0 K 2024-11-20T09:00:15,329 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/B/38552bf1ea2c4929b4d11bafc15b2bf7 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/B/38552bf1ea2c4929b4d11bafc15b2bf7 2024-11-20T09:00:15,333 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-20T09:00:15,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. 2024-11-20T09:00:15,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. as already flushing 2024-11-20T09:00:15,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. 2024-11-20T09:00:15,334 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/B/38552bf1ea2c4929b4d11bafc15b2bf7, entries=400, sequenceid=317, filesize=23.9 K 2024-11-20T09:00:15,333 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => f0e725d6c797c04eb86dfc3be87ab3ba, NAME => 'TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:15,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => f0e725d6c797c04eb86dfc3be87ab3ba, NAME => 'TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:15,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0e725d6c797c04eb86dfc3be87ab3ba, NAME => 'TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0e725d6c797c04eb86dfc3be87ab3ba, NAME => 'TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:15,336 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/C/030d34f5cdcd4e64817eb41c16b5d36c as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/C/030d34f5cdcd4e64817eb41c16b5d36c 2024-11-20T09:00:15,342 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/C/030d34f5cdcd4e64817eb41c16b5d36c, entries=150, sequenceid=317, filesize=12.0 K 2024-11-20T09:00:15,342 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for f0e725d6c797c04eb86dfc3be87ab3ba in 901ms, sequenceid=317, compaction requested=true 2024-11-20T09:00:15,342 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f0e725d6c797c04eb86dfc3be87ab3ba: 2024-11-20T09:00:15,343 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f0e725d6c797c04eb86dfc3be87ab3ba:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T09:00:15,343 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T09:00:15,343 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T09:00:15,343 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f0e725d6c797c04eb86dfc3be87ab3ba:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T09:00:15,343 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T09:00:15,343 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f0e725d6c797c04eb86dfc3be87ab3ba:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T09:00:15,343 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T09:00:15,343 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T09:00:15,344 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 62006 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T09:00:15,344 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 186615 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T09:00:15,345 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1541): f0e725d6c797c04eb86dfc3be87ab3ba/B is initiating minor compaction (all files) 2024-11-20T09:00:15,345 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HStore(1541): f0e725d6c797c04eb86dfc3be87ab3ba/A is initiating minor compaction (all files) 2024-11-20T09:00:15,345 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f0e725d6c797c04eb86dfc3be87ab3ba/B in TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. 2024-11-20T09:00:15,345 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f0e725d6c797c04eb86dfc3be87ab3ba/A in TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. 2024-11-20T09:00:15,345 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/B/7441c27cf52845cfb0214f8acfc69435, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/B/7757629ea3fb4086a471d6baa926888a, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/B/70fc1509b6d84c69b509de5492eabb5c, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/B/38552bf1ea2c4929b4d11bafc15b2bf7] into tmpdir=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp, totalSize=60.6 K 2024-11-20T09:00:15,345 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/A/755ccc014a9e4042855401cc641be14f, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/A/0e5829fa152e4526bda4d103be67f637, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/A/e715ea8b214146aea232c62507b45252, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/A/7421fef9056b4903b6a3c7c36fbacbb7] into tmpdir=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp, totalSize=182.2 K 2024-11-20T09:00:15,345 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. 2024-11-20T09:00:15,345 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. files: [hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/A/755ccc014a9e4042855401cc641be14f, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/A/0e5829fa152e4526bda4d103be67f637, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/A/e715ea8b214146aea232c62507b45252, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/A/7421fef9056b4903b6a3c7c36fbacbb7] 2024-11-20T09:00:15,346 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting 7441c27cf52845cfb0214f8acfc69435, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1732093208184 2024-11-20T09:00:15,346 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.Compactor(225): Compacting 755ccc014a9e4042855401cc641be14f, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1732093208184 2024-11-20T09:00:15,346 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting 7757629ea3fb4086a471d6baa926888a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1732093211570 2024-11-20T09:00:15,346 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting 70fc1509b6d84c69b509de5492eabb5c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=306, earliestPutTs=1732093212203 2024-11-20T09:00:15,347 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting 38552bf1ea2c4929b4d11bafc15b2bf7, keycount=400, bloomtype=ROW, size=23.9 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1732093214423 2024-11-20T09:00:15,347 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0e5829fa152e4526bda4d103be67f637, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1732093211551 2024-11-20T09:00:15,347 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.Compactor(225): Compacting e715ea8b214146aea232c62507b45252, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=306, earliestPutTs=1732093212203 2024-11-20T09:00:15,348 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7421fef9056b4903b6a3c7c36fbacbb7, keycount=400, bloomtype=ROW, size=73.0 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1732093214423 2024-11-20T09:00:15,364 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=f0e725d6c797c04eb86dfc3be87ab3ba] 2024-11-20T09:00:15,367 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f0e725d6c797c04eb86dfc3be87ab3ba#B#compaction#195 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-20T09:00:15,367 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/B/cc5c74fdde1342f6a73c6a87e6793a50 is 50, key is test_row_0/B:col10/1732093214433/Put/seqid=0 2024-11-20T09:00:15,371 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411205da138fbe72b4ac1bff7e3d5b46df7bb_f0e725d6c797c04eb86dfc3be87ab3ba store=[table=TestAcidGuarantees family=A region=f0e725d6c797c04eb86dfc3be87ab3ba] 2024-11-20T09:00:15,374 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411205da138fbe72b4ac1bff7e3d5b46df7bb_f0e725d6c797c04eb86dfc3be87ab3ba, store=[table=TestAcidGuarantees family=A region=f0e725d6c797c04eb86dfc3be87ab3ba] 2024-11-20T09:00:15,375 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411205da138fbe72b4ac1bff7e3d5b46df7bb_f0e725d6c797c04eb86dfc3be87ab3ba because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=f0e725d6c797c04eb86dfc3be87ab3ba] 2024-11-20T09:00:15,382 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742057_1233 (size=13085) 2024-11-20T09:00:15,391 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/B/cc5c74fdde1342f6a73c6a87e6793a50 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/B/cc5c74fdde1342f6a73c6a87e6793a50 2024-11-20T09:00:15,398 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f0e725d6c797c04eb86dfc3be87ab3ba/B of f0e725d6c797c04eb86dfc3be87ab3ba into cc5c74fdde1342f6a73c6a87e6793a50(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T09:00:15,398 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f0e725d6c797c04eb86dfc3be87ab3ba: 2024-11-20T09:00:15,398 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., storeName=f0e725d6c797c04eb86dfc3be87ab3ba/B, priority=12, startTime=1732093215343; duration=0sec 2024-11-20T09:00:15,398 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T09:00:15,398 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f0e725d6c797c04eb86dfc3be87ab3ba:B 2024-11-20T09:00:15,398 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T09:00:15,401 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49768 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T09:00:15,401 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1541): f0e725d6c797c04eb86dfc3be87ab3ba/C is initiating minor compaction (all files) 2024-11-20T09:00:15,401 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f0e725d6c797c04eb86dfc3be87ab3ba/C in TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. 2024-11-20T09:00:15,401 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742058_1234 (size=4469) 2024-11-20T09:00:15,403 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/C/27dbbdc0cebd4201ad0eb2d133f4f2e9, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/C/dc4c8f65ee594a12b6545f4d9667bbff, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/C/8caf0f987d6b4226bd4f4f19cd0fdedd, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/C/030d34f5cdcd4e64817eb41c16b5d36c] into tmpdir=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp, totalSize=48.6 K 2024-11-20T09:00:15,404 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f0e725d6c797c04eb86dfc3be87ab3ba#A#compaction#196 average throughput is 0.61 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T09:00:15,404 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting 27dbbdc0cebd4201ad0eb2d133f4f2e9, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1732093208184 2024-11-20T09:00:15,404 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting dc4c8f65ee594a12b6545f4d9667bbff, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1732093211570 2024-11-20T09:00:15,405 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/A/efd778d30df946f8a463034f813a489c is 175, key is test_row_0/A:col10/1732093214433/Put/seqid=0 2024-11-20T09:00:15,405 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting 8caf0f987d6b4226bd4f4f19cd0fdedd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=306, earliestPutTs=1732093212203 2024-11-20T09:00:15,405 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting 030d34f5cdcd4e64817eb41c16b5d36c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1732093214433 2024-11-20T09:00:15,409 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742059_1235 (size=32005) 2024-11-20T09:00:15,417 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/A/efd778d30df946f8a463034f813a489c as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/A/efd778d30df946f8a463034f813a489c 2024-11-20T09:00:15,422 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f0e725d6c797c04eb86dfc3be87ab3ba#C#compaction#197 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T09:00:15,423 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/C/74d11fcdc21f4e02abae769183a1cbe2 is 50, key is test_row_0/C:col10/1732093214433/Put/seqid=0 2024-11-20T09:00:15,423 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f0e725d6c797c04eb86dfc3be87ab3ba/A of f0e725d6c797c04eb86dfc3be87ab3ba into efd778d30df946f8a463034f813a489c(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T09:00:15,423 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f0e725d6c797c04eb86dfc3be87ab3ba: 2024-11-20T09:00:15,423 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., storeName=f0e725d6c797c04eb86dfc3be87ab3ba/A, priority=12, startTime=1732093215343; duration=0sec 2024-11-20T09:00:15,424 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T09:00:15,424 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f0e725d6c797c04eb86dfc3be87ab3ba:A 2024-11-20T09:00:15,432 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742060_1236 (size=13051) 2024-11-20T09:00:15,487 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-20T09:00:15,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. 2024-11-20T09:00:15,487 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2902): Flushing f0e725d6c797c04eb86dfc3be87ab3ba 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-20T09:00:15,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120d1a99ca3505d495cadf1ecda38802808_f0e725d6c797c04eb86dfc3be87ab3ba is 50, key is test_row_0/A:col10/1732093214503/Put/seqid=0 2024-11-20T09:00:15,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-20T09:00:15,502 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742061_1237 (size=12454) 2024-11-20T09:00:15,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:15,508 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120d1a99ca3505d495cadf1ecda38802808_f0e725d6c797c04eb86dfc3be87ab3ba to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120d1a99ca3505d495cadf1ecda38802808_f0e725d6c797c04eb86dfc3be87ab3ba 2024-11-20T09:00:15,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/A/4f13e5c630ae4c438a34a6af44174dfc, store: [table=TestAcidGuarantees family=A region=f0e725d6c797c04eb86dfc3be87ab3ba] 2024-11-20T09:00:15,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/A/4f13e5c630ae4c438a34a6af44174dfc is 175, key is test_row_0/A:col10/1732093214503/Put/seqid=0 2024-11-20T09:00:15,517 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742062_1238 (size=31255) 2024-11-20T09:00:15,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(8855): Flush requested on f0e725d6c797c04eb86dfc3be87ab3ba 2024-11-20T09:00:15,648 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. as already flushing 2024-11-20T09:00:15,665 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:15,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48450 deadline: 1732093275662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:15,665 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:15,665 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:15,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48388 deadline: 1732093275664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:15,666 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:15,666 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:15,666 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:15,666 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:15,666 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:15,670 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:15,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48466 deadline: 1732093275665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:15,671 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:15,671 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:15,671 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:15,777 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:15,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48450 deadline: 1732093275777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:15,778 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:15,778 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:15,778 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:15,778 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:15,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48388 deadline: 1732093275777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:15,778 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:15,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48466 deadline: 1732093275777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:15,778 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:15,778 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:15,778 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:15,778 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:15,778 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:15,778 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:15,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-20T09:00:15,838 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/C/74d11fcdc21f4e02abae769183a1cbe2 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/C/74d11fcdc21f4e02abae769183a1cbe2 2024-11-20T09:00:15,842 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f0e725d6c797c04eb86dfc3be87ab3ba/C of f0e725d6c797c04eb86dfc3be87ab3ba into 74d11fcdc21f4e02abae769183a1cbe2(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T09:00:15,842 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f0e725d6c797c04eb86dfc3be87ab3ba: 2024-11-20T09:00:15,842 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., storeName=f0e725d6c797c04eb86dfc3be87ab3ba/C, priority=12, startTime=1732093215343; duration=0sec 2024-11-20T09:00:15,843 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T09:00:15,843 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f0e725d6c797c04eb86dfc3be87ab3ba:C 2024-11-20T09:00:15,918 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=344, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/A/4f13e5c630ae4c438a34a6af44174dfc 2024-11-20T09:00:15,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/B/4bfff28f225f494d8c0511ced1f271a5 is 50, key is test_row_0/B:col10/1732093214503/Put/seqid=0 2024-11-20T09:00:15,929 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742063_1239 (size=12301) 2024-11-20T09:00:15,930 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=344 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/B/4bfff28f225f494d8c0511ced1f271a5 2024-11-20T09:00:15,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/C/a89d18a40d674af9b7d7bf1ae5a9dc98 is 50, key is test_row_0/C:col10/1732093214503/Put/seqid=0 2024-11-20T09:00:15,942 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742064_1240 (size=12301) 2024-11-20T09:00:15,942 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=344 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/C/a89d18a40d674af9b7d7bf1ae5a9dc98 2024-11-20T09:00:15,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/A/4f13e5c630ae4c438a34a6af44174dfc as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/A/4f13e5c630ae4c438a34a6af44174dfc 2024-11-20T09:00:15,951 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/A/4f13e5c630ae4c438a34a6af44174dfc, entries=150, sequenceid=344, filesize=30.5 K 2024-11-20T09:00:15,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/B/4bfff28f225f494d8c0511ced1f271a5 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/B/4bfff28f225f494d8c0511ced1f271a5 2024-11-20T09:00:15,960 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/B/4bfff28f225f494d8c0511ced1f271a5, entries=150, sequenceid=344, filesize=12.0 K 2024-11-20T09:00:15,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/C/a89d18a40d674af9b7d7bf1ae5a9dc98 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/C/a89d18a40d674af9b7d7bf1ae5a9dc98 2024-11-20T09:00:15,966 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/C/a89d18a40d674af9b7d7bf1ae5a9dc98, entries=150, sequenceid=344, filesize=12.0 K 2024-11-20T09:00:15,967 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for f0e725d6c797c04eb86dfc3be87ab3ba in 480ms, sequenceid=344, compaction requested=false 2024-11-20T09:00:15,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2603): Flush status journal for f0e725d6c797c04eb86dfc3be87ab3ba: 2024-11-20T09:00:15,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. 2024-11-20T09:00:15,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=47 2024-11-20T09:00:15,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.HMaster(4169): Remote procedure done, pid=47 2024-11-20T09:00:15,970 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=46 2024-11-20T09:00:15,970 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 788 msec 2024-11-20T09:00:15,971 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees in 793 msec 2024-11-20T09:00:15,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(8855): Flush requested on f0e725d6c797c04eb86dfc3be87ab3ba 2024-11-20T09:00:15,988 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T09:00:15,988 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f0e725d6c797c04eb86dfc3be87ab3ba 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-20T09:00:16,022 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112086509b49a07849dba8dfb5f9bf418f94_f0e725d6c797c04eb86dfc3be87ab3ba is 50, key is test_row_0/A:col10/1732093215987/Put/seqid=0 2024-11-20T09:00:16,029 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742065_1241 (size=12454) 2024-11-20T09:00:16,056 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:16,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48466 deadline: 1732093276047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:16,056 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:16,056 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:16,056 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:16,057 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:16,057 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:16,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48450 deadline: 1732093276053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:16,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48388 deadline: 1732093276056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:16,058 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:16,058 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:16,058 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:16,058 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:16,058 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:16,058 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:16,169 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:16,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48466 deadline: 1732093276166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:16,169 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:16,169 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:16,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48450 deadline: 1732093276167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:16,169 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:16,169 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:16,170 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:16,170 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:16,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48388 deadline: 1732093276167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:16,170 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:16,170 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:16,170 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:16,170 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:16,170 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:16,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-20T09:00:16,317 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T09:00:16,318 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T09:00:16,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] procedure2.ProcedureExecutor(1139): Stored pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees 2024-11-20T09:00:16,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-20T09:00:16,319 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T09:00:16,320 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T09:00:16,320 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T09:00:16,379 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:16,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48466 deadline: 1732093276376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:16,380 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:16,380 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:16,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48450 deadline: 1732093276377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:16,380 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:16,380 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:16,380 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:16,380 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:16,380 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:16,380 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:16,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48388 deadline: 1732093276377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:16,380 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:16,381 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:16,381 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:16,397 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:16,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48420 deadline: 1732093276397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:16,398 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to be1eb2620e0e,34223,1732093159960 for 'test_row_1' in f0e725d6c797c04eb86dfc3be87ab3ba of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8212 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-20T09:00:16,399 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:16,399 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:16,399 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:16,422 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:16,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48394 deadline: 1732093276417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:16,422 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to be1eb2620e0e,34223,1732093159960 for 'test_row_1' in f0e725d6c797c04eb86dfc3be87ab3ba of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8243 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-20T09:00:16,422 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:16,423 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:16,423 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:16,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-20T09:00:16,430 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:16,434 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112086509b49a07849dba8dfb5f9bf418f94_f0e725d6c797c04eb86dfc3be87ab3ba to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112086509b49a07849dba8dfb5f9bf418f94_f0e725d6c797c04eb86dfc3be87ab3ba 2024-11-20T09:00:16,435 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/A/9a4cdb9f4c4c4fe2b6ac254ea45f9ed5, store: [table=TestAcidGuarantees family=A region=f0e725d6c797c04eb86dfc3be87ab3ba] 2024-11-20T09:00:16,436 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/A/9a4cdb9f4c4c4fe2b6ac254ea45f9ed5 is 175, key is test_row_0/A:col10/1732093215987/Put/seqid=0 2024-11-20T09:00:16,440 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742066_1242 (size=31255) 2024-11-20T09:00:16,472 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-20T09:00:16,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. 2024-11-20T09:00:16,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. as already flushing 2024-11-20T09:00:16,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. 2024-11-20T09:00:16,473 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => f0e725d6c797c04eb86dfc3be87ab3ba, NAME => 'TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:16,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => f0e725d6c797c04eb86dfc3be87ab3ba, NAME => 'TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:16,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0e725d6c797c04eb86dfc3be87ab3ba, NAME => 'TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0e725d6c797c04eb86dfc3be87ab3ba, NAME => 'TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:16,626 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-20T09:00:16,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. 2024-11-20T09:00:16,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. as already flushing 2024-11-20T09:00:16,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. 2024-11-20T09:00:16,626 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => f0e725d6c797c04eb86dfc3be87ab3ba, NAME => 'TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:16,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => f0e725d6c797c04eb86dfc3be87ab3ba, NAME => 'TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:16,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0e725d6c797c04eb86dfc3be87ab3ba, NAME => 'TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0e725d6c797c04eb86dfc3be87ab3ba, NAME => 'TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:16,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-20T09:00:16,689 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:16,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48466 deadline: 1732093276687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:16,689 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:16,689 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:16,689 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:16,690 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:16,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48450 deadline: 1732093276687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:16,690 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:16,690 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:16,690 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:16,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48388 deadline: 1732093276687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:16,690 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:16,690 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:16,690 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:16,690 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:16,779 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-20T09:00:16,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. 2024-11-20T09:00:16,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. as already flushing 2024-11-20T09:00:16,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. 2024-11-20T09:00:16,779 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => f0e725d6c797c04eb86dfc3be87ab3ba, NAME => 'TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:16,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => f0e725d6c797c04eb86dfc3be87ab3ba, NAME => 'TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:16,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0e725d6c797c04eb86dfc3be87ab3ba, NAME => 'TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0e725d6c797c04eb86dfc3be87ab3ba, NAME => 'TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:16,841 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=358, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/A/9a4cdb9f4c4c4fe2b6ac254ea45f9ed5 2024-11-20T09:00:16,849 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/B/768af76c0f164e799868e867bd05a1a1 is 50, key is test_row_0/B:col10/1732093215987/Put/seqid=0 2024-11-20T09:00:16,853 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742067_1243 (size=12301) 2024-11-20T09:00:16,854 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=358 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/B/768af76c0f164e799868e867bd05a1a1 2024-11-20T09:00:16,862 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/C/decb431db6644214bc25ed01c7cd1fd0 is 50, key is test_row_0/C:col10/1732093215987/Put/seqid=0 2024-11-20T09:00:16,866 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742068_1244 (size=12301) 2024-11-20T09:00:16,932 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-20T09:00:16,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. 2024-11-20T09:00:16,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. as already flushing 2024-11-20T09:00:16,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. 2024-11-20T09:00:16,933 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => f0e725d6c797c04eb86dfc3be87ab3ba, NAME => 'TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:16,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => f0e725d6c797c04eb86dfc3be87ab3ba, NAME => 'TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:16,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0e725d6c797c04eb86dfc3be87ab3ba, NAME => 'TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0e725d6c797c04eb86dfc3be87ab3ba, NAME => 'TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:16,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-20T09:00:17,086 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-20T09:00:17,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. 2024-11-20T09:00:17,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. as already flushing 2024-11-20T09:00:17,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. 2024-11-20T09:00:17,086 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => f0e725d6c797c04eb86dfc3be87ab3ba, NAME => 'TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:17,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => f0e725d6c797c04eb86dfc3be87ab3ba, NAME => 'TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:17,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0e725d6c797c04eb86dfc3be87ab3ba, NAME => 'TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0e725d6c797c04eb86dfc3be87ab3ba, NAME => 'TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:17,199 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:17,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48466 deadline: 1732093277197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:17,199 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:17,199 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:17,199 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:17,200 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:17,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48450 deadline: 1732093277197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:17,200 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:17,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48388 deadline: 1732093277197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:17,200 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:17,200 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:17,200 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:17,200 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:17,200 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:17,200 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:17,239 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-20T09:00:17,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. 2024-11-20T09:00:17,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. as already flushing 2024-11-20T09:00:17,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. 2024-11-20T09:00:17,239 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => f0e725d6c797c04eb86dfc3be87ab3ba, NAME => 'TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:17,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => f0e725d6c797c04eb86dfc3be87ab3ba, NAME => 'TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:17,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0e725d6c797c04eb86dfc3be87ab3ba, NAME => 'TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0e725d6c797c04eb86dfc3be87ab3ba, NAME => 'TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:17,267 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=358 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/C/decb431db6644214bc25ed01c7cd1fd0 2024-11-20T09:00:17,272 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/A/9a4cdb9f4c4c4fe2b6ac254ea45f9ed5 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/A/9a4cdb9f4c4c4fe2b6ac254ea45f9ed5 2024-11-20T09:00:17,276 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/A/9a4cdb9f4c4c4fe2b6ac254ea45f9ed5, entries=150, sequenceid=358, filesize=30.5 K 2024-11-20T09:00:17,277 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/B/768af76c0f164e799868e867bd05a1a1 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/B/768af76c0f164e799868e867bd05a1a1 2024-11-20T09:00:17,282 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/B/768af76c0f164e799868e867bd05a1a1, entries=150, sequenceid=358, filesize=12.0 K 2024-11-20T09:00:17,283 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/C/decb431db6644214bc25ed01c7cd1fd0 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/C/decb431db6644214bc25ed01c7cd1fd0 2024-11-20T09:00:17,288 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/C/decb431db6644214bc25ed01c7cd1fd0, entries=150, sequenceid=358, filesize=12.0 K 2024-11-20T09:00:17,289 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for f0e725d6c797c04eb86dfc3be87ab3ba in 1300ms, sequenceid=358, compaction requested=true 2024-11-20T09:00:17,289 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f0e725d6c797c04eb86dfc3be87ab3ba: 2024-11-20T09:00:17,289 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f0e725d6c797c04eb86dfc3be87ab3ba:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T09:00:17,289 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T09:00:17,289 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T09:00:17,289 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f0e725d6c797c04eb86dfc3be87ab3ba:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T09:00:17,289 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T09:00:17,289 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f0e725d6c797c04eb86dfc3be87ab3ba:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T09:00:17,289 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T09:00:17,289 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T09:00:17,290 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94515 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T09:00:17,290 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T09:00:17,290 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HStore(1541): f0e725d6c797c04eb86dfc3be87ab3ba/A is initiating minor compaction (all files) 2024-11-20T09:00:17,290 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1541): f0e725d6c797c04eb86dfc3be87ab3ba/B is initiating minor compaction (all files) 2024-11-20T09:00:17,290 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f0e725d6c797c04eb86dfc3be87ab3ba/A in TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. 2024-11-20T09:00:17,290 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f0e725d6c797c04eb86dfc3be87ab3ba/B in TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. 2024-11-20T09:00:17,290 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/A/efd778d30df946f8a463034f813a489c, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/A/4f13e5c630ae4c438a34a6af44174dfc, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/A/9a4cdb9f4c4c4fe2b6ac254ea45f9ed5] into tmpdir=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp, totalSize=92.3 K 2024-11-20T09:00:17,290 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/B/cc5c74fdde1342f6a73c6a87e6793a50, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/B/4bfff28f225f494d8c0511ced1f271a5, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/B/768af76c0f164e799868e867bd05a1a1] into tmpdir=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp, totalSize=36.8 K 2024-11-20T09:00:17,290 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. 2024-11-20T09:00:17,290 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. files: [hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/A/efd778d30df946f8a463034f813a489c, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/A/4f13e5c630ae4c438a34a6af44174dfc, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/A/9a4cdb9f4c4c4fe2b6ac254ea45f9ed5] 2024-11-20T09:00:17,291 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting cc5c74fdde1342f6a73c6a87e6793a50, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1732093214433 2024-11-20T09:00:17,291 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.Compactor(225): Compacting efd778d30df946f8a463034f813a489c, keycount=150, bloomtype=ROW, size=31.3 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1732093214433 2024-11-20T09:00:17,291 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting 4bfff28f225f494d8c0511ced1f271a5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=344, earliestPutTs=1732093214501 2024-11-20T09:00:17,292 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4f13e5c630ae4c438a34a6af44174dfc, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=344, earliestPutTs=1732093214501 2024-11-20T09:00:17,292 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting 768af76c0f164e799868e867bd05a1a1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=358, earliestPutTs=1732093215664 2024-11-20T09:00:17,292 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9a4cdb9f4c4c4fe2b6ac254ea45f9ed5, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=358, earliestPutTs=1732093215664 2024-11-20T09:00:17,299 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=f0e725d6c797c04eb86dfc3be87ab3ba] 2024-11-20T09:00:17,302 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f0e725d6c797c04eb86dfc3be87ab3ba#B#compaction#205 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-20T09:00:17,303 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/B/bde25d9be5614b92bbac7d32df128b16 is 50, key is test_row_0/B:col10/1732093215987/Put/seqid=0 2024-11-20T09:00:17,304 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241120aac308091ed84f9189b86c0e79f347d0_f0e725d6c797c04eb86dfc3be87ab3ba store=[table=TestAcidGuarantees family=A region=f0e725d6c797c04eb86dfc3be87ab3ba] 2024-11-20T09:00:17,306 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241120aac308091ed84f9189b86c0e79f347d0_f0e725d6c797c04eb86dfc3be87ab3ba, store=[table=TestAcidGuarantees family=A region=f0e725d6c797c04eb86dfc3be87ab3ba] 2024-11-20T09:00:17,306 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120aac308091ed84f9189b86c0e79f347d0_f0e725d6c797c04eb86dfc3be87ab3ba because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=f0e725d6c797c04eb86dfc3be87ab3ba] 2024-11-20T09:00:17,338 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742070_1246 (size=4469) 2024-11-20T09:00:17,338 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742069_1245 (size=13187) 2024-11-20T09:00:17,344 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/B/bde25d9be5614b92bbac7d32df128b16 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/B/bde25d9be5614b92bbac7d32df128b16 2024-11-20T09:00:17,351 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f0e725d6c797c04eb86dfc3be87ab3ba/B of f0e725d6c797c04eb86dfc3be87ab3ba into bde25d9be5614b92bbac7d32df128b16(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T09:00:17,351 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f0e725d6c797c04eb86dfc3be87ab3ba: 2024-11-20T09:00:17,351 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., storeName=f0e725d6c797c04eb86dfc3be87ab3ba/B, priority=13, startTime=1732093217289; duration=0sec 2024-11-20T09:00:17,352 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T09:00:17,352 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f0e725d6c797c04eb86dfc3be87ab3ba:B 2024-11-20T09:00:17,352 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T09:00:17,353 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T09:00:17,353 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1541): f0e725d6c797c04eb86dfc3be87ab3ba/C is initiating minor compaction (all files) 2024-11-20T09:00:17,353 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f0e725d6c797c04eb86dfc3be87ab3ba/C in TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. 2024-11-20T09:00:17,354 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/C/74d11fcdc21f4e02abae769183a1cbe2, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/C/a89d18a40d674af9b7d7bf1ae5a9dc98, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/C/decb431db6644214bc25ed01c7cd1fd0] into tmpdir=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp, totalSize=36.8 K 2024-11-20T09:00:17,354 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting 74d11fcdc21f4e02abae769183a1cbe2, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1732093214433 2024-11-20T09:00:17,355 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting a89d18a40d674af9b7d7bf1ae5a9dc98, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=344, earliestPutTs=1732093214501 2024-11-20T09:00:17,356 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting decb431db6644214bc25ed01c7cd1fd0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=358, earliestPutTs=1732093215664 2024-11-20T09:00:17,367 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f0e725d6c797c04eb86dfc3be87ab3ba#C#compaction#206 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-20T09:00:17,368 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/C/b69bfcbd2db842a7b1f5468d6e073679 is 50, key is test_row_0/C:col10/1732093215987/Put/seqid=0 2024-11-20T09:00:17,376 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742071_1247 (size=13153) 2024-11-20T09:00:17,392 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-20T09:00:17,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. 2024-11-20T09:00:17,393 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2902): Flushing f0e725d6c797c04eb86dfc3be87ab3ba 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-20T09:00:17,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120890d266f803c4681aaf1ffe741598916_f0e725d6c797c04eb86dfc3be87ab3ba is 50, key is test_row_0/A:col10/1732093216045/Put/seqid=0 2024-11-20T09:00:17,409 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742072_1248 (size=12454) 2024-11-20T09:00:17,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-20T09:00:17,739 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f0e725d6c797c04eb86dfc3be87ab3ba#A#compaction#204 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T09:00:17,739 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/A/fd4fd714f14047628d03b42f0ef5337b is 175, key is test_row_0/A:col10/1732093215987/Put/seqid=0 2024-11-20T09:00:17,745 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742073_1249 (size=32107) 2024-11-20T09:00:17,782 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/C/b69bfcbd2db842a7b1f5468d6e073679 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/C/b69bfcbd2db842a7b1f5468d6e073679 2024-11-20T09:00:17,789 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f0e725d6c797c04eb86dfc3be87ab3ba/C of f0e725d6c797c04eb86dfc3be87ab3ba into b69bfcbd2db842a7b1f5468d6e073679(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T09:00:17,789 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f0e725d6c797c04eb86dfc3be87ab3ba: 2024-11-20T09:00:17,789 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., storeName=f0e725d6c797c04eb86dfc3be87ab3ba/C, priority=13, startTime=1732093217289; duration=0sec 2024-11-20T09:00:17,789 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T09:00:17,789 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f0e725d6c797c04eb86dfc3be87ab3ba:C 2024-11-20T09:00:17,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,814 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120890d266f803c4681aaf1ffe741598916_f0e725d6c797c04eb86dfc3be87ab3ba to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120890d266f803c4681aaf1ffe741598916_f0e725d6c797c04eb86dfc3be87ab3ba 2024-11-20T09:00:17,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/A/9e5ea3e5bc124b69bd279d8168824cc1, store: [table=TestAcidGuarantees family=A region=f0e725d6c797c04eb86dfc3be87ab3ba] 2024-11-20T09:00:17,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/A/9e5ea3e5bc124b69bd279d8168824cc1 is 175, key is test_row_0/A:col10/1732093216045/Put/seqid=0 2024-11-20T09:00:17,821 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742074_1250 (size=31255) 2024-11-20T09:00:17,822 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=382, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/A/9e5ea3e5bc124b69bd279d8168824cc1 2024-11-20T09:00:17,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/B/d476a20296da4aa2ab0b702cb569a05c is 50, key is test_row_0/B:col10/1732093216045/Put/seqid=0 2024-11-20T09:00:17,844 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742075_1251 (size=12301) 2024-11-20T09:00:17,846 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=382 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/B/d476a20296da4aa2ab0b702cb569a05c 2024-11-20T09:00:17,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/C/05a47f98435046c696309bb92ba289d1 is 50, key is test_row_0/C:col10/1732093216045/Put/seqid=0 2024-11-20T09:00:17,865 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742076_1252 (size=12301) 2024-11-20T09:00:17,866 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=382 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/C/05a47f98435046c696309bb92ba289d1 2024-11-20T09:00:17,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/A/9e5ea3e5bc124b69bd279d8168824cc1 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/A/9e5ea3e5bc124b69bd279d8168824cc1 2024-11-20T09:00:17,878 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/A/9e5ea3e5bc124b69bd279d8168824cc1, entries=150, sequenceid=382, filesize=30.5 K 2024-11-20T09:00:17,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/B/d476a20296da4aa2ab0b702cb569a05c as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/B/d476a20296da4aa2ab0b702cb569a05c 2024-11-20T09:00:17,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,883 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/B/d476a20296da4aa2ab0b702cb569a05c, entries=150, sequenceid=382, filesize=12.0 K 2024-11-20T09:00:17,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/C/05a47f98435046c696309bb92ba289d1 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/C/05a47f98435046c696309bb92ba289d1 2024-11-20T09:00:17,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,891 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/C/05a47f98435046c696309bb92ba289d1, entries=150, sequenceid=382, filesize=12.0 K 2024-11-20T09:00:17,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,892 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=0 B/0 for f0e725d6c797c04eb86dfc3be87ab3ba in 499ms, sequenceid=382, compaction requested=false 2024-11-20T09:00:17,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2603): Flush status journal for f0e725d6c797c04eb86dfc3be87ab3ba: 2024-11-20T09:00:17,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. 2024-11-20T09:00:17,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=49 2024-11-20T09:00:17,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.HMaster(4169): Remote procedure done, pid=49 2024-11-20T09:00:17,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,896 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=48 2024-11-20T09:00:17,896 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5740 sec 2024-11-20T09:00:17,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,898 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees in 1.5790 sec 2024-11-20T09:00:17,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:17,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,152 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/A/fd4fd714f14047628d03b42f0ef5337b as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/A/fd4fd714f14047628d03b42f0ef5337b 2024-11-20T09:00:18,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,159 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f0e725d6c797c04eb86dfc3be87ab3ba/A of f0e725d6c797c04eb86dfc3be87ab3ba into fd4fd714f14047628d03b42f0ef5337b(size=31.4 K), total size for store is 61.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T09:00:18,159 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f0e725d6c797c04eb86dfc3be87ab3ba: 2024-11-20T09:00:18,159 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., storeName=f0e725d6c797c04eb86dfc3be87ab3ba/A, priority=13, startTime=1732093217289; duration=0sec 2024-11-20T09:00:18,160 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T09:00:18,160 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f0e725d6c797c04eb86dfc3be87ab3ba:A 2024-11-20T09:00:18,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,192 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-20T09:00:18,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(8855): Flush requested on f0e725d6c797c04eb86dfc3be87ab3ba 2024-11-20T09:00:18,272 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T09:00:18,272 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f0e725d6c797c04eb86dfc3be87ab3ba 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-20T09:00:18,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,292 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411208422843b2dcd49caa696d9fbfbb50afe_f0e725d6c797c04eb86dfc3be87ab3ba is 50, key is test_row_1/A:col10/1732093218270/Put/seqid=0 2024-11-20T09:00:18,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,322 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742078_1254 (size=25158) 2024-11-20T09:00:18,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,371 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:18,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48450 deadline: 1732093278366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:18,371 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:18,371 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:18,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48388 deadline: 1732093278370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:18,371 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:18,371 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:18,372 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:18,372 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:18,372 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:18,374 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:18,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48466 deadline: 1732093278373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:18,375 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:18,375 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:18,375 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:18,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-20T09:00:18,467 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T09:00:18,468 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T09:00:18,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] procedure2.ProcedureExecutor(1139): Stored pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees 2024-11-20T09:00:18,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-20T09:00:18,470 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T09:00:18,471 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T09:00:18,471 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T09:00:18,479 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:18,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48450 deadline: 1732093278477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:18,479 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:18,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48388 deadline: 1732093278477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:18,480 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:18,480 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:18,480 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:18,480 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:18,480 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:18,480 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:18,487 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:18,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48466 deadline: 1732093278486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:18,488 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:18,488 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:18,488 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:18,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-20T09:00:18,623 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-20T09:00:18,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. 2024-11-20T09:00:18,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. as already flushing 2024-11-20T09:00:18,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. 2024-11-20T09:00:18,623 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => f0e725d6c797c04eb86dfc3be87ab3ba, NAME => 'TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:18,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => f0e725d6c797c04eb86dfc3be87ab3ba, NAME => 'TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:18,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0e725d6c797c04eb86dfc3be87ab3ba, NAME => 'TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0e725d6c797c04eb86dfc3be87ab3ba, NAME => 'TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:18,688 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:18,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48450 deadline: 1732093278687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:18,689 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:18,689 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:18,689 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:18,689 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:18,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48388 deadline: 1732093278687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:18,689 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:18,689 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:18,689 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:18,699 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:18,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48466 deadline: 1732093278697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:18,700 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:18,700 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:18,700 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:18,723 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:18,727 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411208422843b2dcd49caa696d9fbfbb50afe_f0e725d6c797c04eb86dfc3be87ab3ba to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411208422843b2dcd49caa696d9fbfbb50afe_f0e725d6c797c04eb86dfc3be87ab3ba 2024-11-20T09:00:18,728 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/A/188ae37a340e4059a7c4a2febbed5cc3, store: [table=TestAcidGuarantees family=A region=f0e725d6c797c04eb86dfc3be87ab3ba] 2024-11-20T09:00:18,729 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/A/188ae37a340e4059a7c4a2febbed5cc3 is 175, key is test_row_1/A:col10/1732093218270/Put/seqid=0 2024-11-20T09:00:18,731 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742077_1253 (size=74790) 2024-11-20T09:00:18,731 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=395, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/A/188ae37a340e4059a7c4a2febbed5cc3 2024-11-20T09:00:18,738 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/B/7835282314de4adb95ec1828fcdabbef is 50, key is test_row_1/B:col10/1732093218270/Put/seqid=0 2024-11-20T09:00:18,742 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742079_1255 (size=9857) 2024-11-20T09:00:18,776 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-20T09:00:18,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. 2024-11-20T09:00:18,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. as already flushing 2024-11-20T09:00:18,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. 2024-11-20T09:00:18,777 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => f0e725d6c797c04eb86dfc3be87ab3ba, NAME => 'TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:18,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => f0e725d6c797c04eb86dfc3be87ab3ba, NAME => 'TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:18,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0e725d6c797c04eb86dfc3be87ab3ba, NAME => 'TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0e725d6c797c04eb86dfc3be87ab3ba, NAME => 'TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:18,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-20T09:00:18,930 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-20T09:00:18,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. 2024-11-20T09:00:18,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. as already flushing 2024-11-20T09:00:18,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. 2024-11-20T09:00:18,930 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => f0e725d6c797c04eb86dfc3be87ab3ba, NAME => 'TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:18,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => f0e725d6c797c04eb86dfc3be87ab3ba, NAME => 'TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:18,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0e725d6c797c04eb86dfc3be87ab3ba, NAME => 'TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0e725d6c797c04eb86dfc3be87ab3ba, NAME => 'TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:19,000 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:19,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48450 deadline: 1732093278996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:19,000 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:19,000 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:19,000 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:19,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48388 deadline: 1732093278996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:19,000 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:19,000 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:19,000 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:19,000 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:19,007 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:19,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48466 deadline: 1732093279007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:19,007 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:19,007 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:19,007 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:19,083 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-20T09:00:19,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. 2024-11-20T09:00:19,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. as already flushing 2024-11-20T09:00:19,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. 2024-11-20T09:00:19,084 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => f0e725d6c797c04eb86dfc3be87ab3ba, NAME => 'TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:19,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => f0e725d6c797c04eb86dfc3be87ab3ba, NAME => 'TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:19,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0e725d6c797c04eb86dfc3be87ab3ba, NAME => 'TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0e725d6c797c04eb86dfc3be87ab3ba, NAME => 'TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:19,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-20T09:00:19,143 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=395 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/B/7835282314de4adb95ec1828fcdabbef 2024-11-20T09:00:19,150 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/C/db43dd2d310a4435a7b01da49c075af6 is 50, key is test_row_1/C:col10/1732093218270/Put/seqid=0 2024-11-20T09:00:19,162 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742080_1256 (size=9857) 2024-11-20T09:00:19,236 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-20T09:00:19,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. 2024-11-20T09:00:19,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. as already flushing 2024-11-20T09:00:19,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. 2024-11-20T09:00:19,237 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => f0e725d6c797c04eb86dfc3be87ab3ba, NAME => 'TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:19,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => f0e725d6c797c04eb86dfc3be87ab3ba, NAME => 'TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:19,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0e725d6c797c04eb86dfc3be87ab3ba, NAME => 'TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0e725d6c797c04eb86dfc3be87ab3ba, NAME => 'TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:19,389 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-20T09:00:19,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. 2024-11-20T09:00:19,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. as already flushing 2024-11-20T09:00:19,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. 2024-11-20T09:00:19,390 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => f0e725d6c797c04eb86dfc3be87ab3ba, NAME => 'TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:19,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => f0e725d6c797c04eb86dfc3be87ab3ba, NAME => 'TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:19,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0e725d6c797c04eb86dfc3be87ab3ba, NAME => 'TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0e725d6c797c04eb86dfc3be87ab3ba, NAME => 'TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:19,508 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:19,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48450 deadline: 1732093279507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:19,508 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:19,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48388 deadline: 1732093279507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:19,508 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:19,508 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:19,508 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:19,508 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:19,508 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:19,508 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:19,520 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:19,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48466 deadline: 1732093279517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:19,521 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:19,521 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:19,521 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:19,543 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-20T09:00:19,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. 2024-11-20T09:00:19,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. as already flushing 2024-11-20T09:00:19,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. 2024-11-20T09:00:19,543 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => f0e725d6c797c04eb86dfc3be87ab3ba, NAME => 'TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:19,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => f0e725d6c797c04eb86dfc3be87ab3ba, NAME => 'TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:19,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0e725d6c797c04eb86dfc3be87ab3ba, NAME => 'TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0e725d6c797c04eb86dfc3be87ab3ba, NAME => 'TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:19,563 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=395 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/C/db43dd2d310a4435a7b01da49c075af6 2024-11-20T09:00:19,577 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/A/188ae37a340e4059a7c4a2febbed5cc3 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/A/188ae37a340e4059a7c4a2febbed5cc3 2024-11-20T09:00:19,582 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/A/188ae37a340e4059a7c4a2febbed5cc3, entries=400, sequenceid=395, filesize=73.0 K 2024-11-20T09:00:19,584 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/B/7835282314de4adb95ec1828fcdabbef as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/B/7835282314de4adb95ec1828fcdabbef 2024-11-20T09:00:19,599 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/B/7835282314de4adb95ec1828fcdabbef, entries=100, sequenceid=395, filesize=9.6 K 2024-11-20T09:00:19,600 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/C/db43dd2d310a4435a7b01da49c075af6 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/C/db43dd2d310a4435a7b01da49c075af6 2024-11-20T09:00:19,604 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/C/db43dd2d310a4435a7b01da49c075af6, entries=100, sequenceid=395, filesize=9.6 K 2024-11-20T09:00:19,605 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=154.31 KB/158010 for f0e725d6c797c04eb86dfc3be87ab3ba in 1333ms, sequenceid=395, compaction requested=true 2024-11-20T09:00:19,605 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f0e725d6c797c04eb86dfc3be87ab3ba: 2024-11-20T09:00:19,605 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T09:00:19,606 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f0e725d6c797c04eb86dfc3be87ab3ba:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T09:00:19,606 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T09:00:19,606 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f0e725d6c797c04eb86dfc3be87ab3ba:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T09:00:19,606 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T09:00:19,606 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f0e725d6c797c04eb86dfc3be87ab3ba:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T09:00:19,606 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T09:00:19,606 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T09:00:19,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-20T09:00:19,609 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 138152 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T09:00:19,609 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HStore(1541): f0e725d6c797c04eb86dfc3be87ab3ba/A is initiating minor compaction (all files) 2024-11-20T09:00:19,609 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f0e725d6c797c04eb86dfc3be87ab3ba/A in TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. 2024-11-20T09:00:19,609 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/A/fd4fd714f14047628d03b42f0ef5337b, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/A/9e5ea3e5bc124b69bd279d8168824cc1, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/A/188ae37a340e4059a7c4a2febbed5cc3] into tmpdir=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp, totalSize=134.9 K 2024-11-20T09:00:19,609 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. 2024-11-20T09:00:19,609 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. files: [hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/A/fd4fd714f14047628d03b42f0ef5337b, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/A/9e5ea3e5bc124b69bd279d8168824cc1, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/A/188ae37a340e4059a7c4a2febbed5cc3] 2024-11-20T09:00:19,609 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.Compactor(225): Compacting fd4fd714f14047628d03b42f0ef5337b, keycount=150, bloomtype=ROW, size=31.4 K, encoding=NONE, compression=NONE, seqNum=358, earliestPutTs=1732093215664 2024-11-20T09:00:19,610 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35345 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T09:00:19,610 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1541): f0e725d6c797c04eb86dfc3be87ab3ba/B is initiating minor compaction (all files) 2024-11-20T09:00:19,610 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f0e725d6c797c04eb86dfc3be87ab3ba/B in TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. 2024-11-20T09:00:19,610 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/B/bde25d9be5614b92bbac7d32df128b16, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/B/d476a20296da4aa2ab0b702cb569a05c, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/B/7835282314de4adb95ec1828fcdabbef] into tmpdir=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp, totalSize=34.5 K 2024-11-20T09:00:19,610 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9e5ea3e5bc124b69bd279d8168824cc1, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=382, earliestPutTs=1732093216045 2024-11-20T09:00:19,610 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting bde25d9be5614b92bbac7d32df128b16, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=358, earliestPutTs=1732093215664 2024-11-20T09:00:19,611 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting d476a20296da4aa2ab0b702cb569a05c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=382, earliestPutTs=1732093216045 2024-11-20T09:00:19,611 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.Compactor(225): Compacting 188ae37a340e4059a7c4a2febbed5cc3, keycount=400, bloomtype=ROW, size=73.0 K, encoding=NONE, compression=NONE, seqNum=395, earliestPutTs=1732093218232 2024-11-20T09:00:19,612 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting 7835282314de4adb95ec1828fcdabbef, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=395, earliestPutTs=1732093218245 2024-11-20T09:00:19,618 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=f0e725d6c797c04eb86dfc3be87ab3ba] 2024-11-20T09:00:19,620 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f0e725d6c797c04eb86dfc3be87ab3ba#B#compaction#214 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-20T09:00:19,621 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024112026619f16e8534ebbb8141b8b6bb97958_f0e725d6c797c04eb86dfc3be87ab3ba store=[table=TestAcidGuarantees family=A region=f0e725d6c797c04eb86dfc3be87ab3ba] 2024-11-20T09:00:19,621 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/B/03b3112046764f18ba8cbce9ccedc3fd is 50, key is test_row_0/B:col10/1732093216045/Put/seqid=0 2024-11-20T09:00:19,623 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024112026619f16e8534ebbb8141b8b6bb97958_f0e725d6c797c04eb86dfc3be87ab3ba, store=[table=TestAcidGuarantees family=A region=f0e725d6c797c04eb86dfc3be87ab3ba] 2024-11-20T09:00:19,623 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112026619f16e8534ebbb8141b8b6bb97958_f0e725d6c797c04eb86dfc3be87ab3ba because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=f0e725d6c797c04eb86dfc3be87ab3ba] 2024-11-20T09:00:19,631 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742081_1257 (size=4469) 2024-11-20T09:00:19,635 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f0e725d6c797c04eb86dfc3be87ab3ba#A#compaction#213 average throughput is 1.44 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T09:00:19,635 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/A/dbc3195e2bd24ef4a99fcc8ae70e8167 is 175, key is test_row_0/A:col10/1732093216045/Put/seqid=0 2024-11-20T09:00:19,636 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742082_1258 (size=13289) 2024-11-20T09:00:19,645 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/B/03b3112046764f18ba8cbce9ccedc3fd as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/B/03b3112046764f18ba8cbce9ccedc3fd 2024-11-20T09:00:19,653 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f0e725d6c797c04eb86dfc3be87ab3ba/B of f0e725d6c797c04eb86dfc3be87ab3ba into 03b3112046764f18ba8cbce9ccedc3fd(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T09:00:19,653 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f0e725d6c797c04eb86dfc3be87ab3ba: 2024-11-20T09:00:19,653 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., storeName=f0e725d6c797c04eb86dfc3be87ab3ba/B, priority=13, startTime=1732093219606; duration=0sec 2024-11-20T09:00:19,653 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T09:00:19,653 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f0e725d6c797c04eb86dfc3be87ab3ba:B 2024-11-20T09:00:19,653 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T09:00:19,654 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35311 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T09:00:19,654 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1541): f0e725d6c797c04eb86dfc3be87ab3ba/C is initiating minor compaction (all files) 2024-11-20T09:00:19,654 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f0e725d6c797c04eb86dfc3be87ab3ba/C in TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. 2024-11-20T09:00:19,655 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/C/b69bfcbd2db842a7b1f5468d6e073679, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/C/05a47f98435046c696309bb92ba289d1, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/C/db43dd2d310a4435a7b01da49c075af6] into tmpdir=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp, totalSize=34.5 K 2024-11-20T09:00:19,655 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting b69bfcbd2db842a7b1f5468d6e073679, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=358, earliestPutTs=1732093215664 2024-11-20T09:00:19,656 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting 05a47f98435046c696309bb92ba289d1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=382, earliestPutTs=1732093216045 2024-11-20T09:00:19,656 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting db43dd2d310a4435a7b01da49c075af6, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=395, earliestPutTs=1732093218245 2024-11-20T09:00:19,670 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742083_1259 (size=32316) 2024-11-20T09:00:19,680 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/A/dbc3195e2bd24ef4a99fcc8ae70e8167 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/A/dbc3195e2bd24ef4a99fcc8ae70e8167 2024-11-20T09:00:19,687 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f0e725d6c797c04eb86dfc3be87ab3ba/A of f0e725d6c797c04eb86dfc3be87ab3ba into dbc3195e2bd24ef4a99fcc8ae70e8167(size=31.6 K), total size for store is 31.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T09:00:19,687 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f0e725d6c797c04eb86dfc3be87ab3ba: 2024-11-20T09:00:19,687 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., storeName=f0e725d6c797c04eb86dfc3be87ab3ba/A, priority=13, startTime=1732093219605; duration=0sec 2024-11-20T09:00:19,688 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T09:00:19,688 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f0e725d6c797c04eb86dfc3be87ab3ba:A 2024-11-20T09:00:19,691 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f0e725d6c797c04eb86dfc3be87ab3ba#C#compaction#215 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T09:00:19,692 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/C/e54aba70aa4e49c4a3d13f361ee884ef is 50, key is test_row_0/C:col10/1732093216045/Put/seqid=0 2024-11-20T09:00:19,697 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-20T09:00:19,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. 2024-11-20T09:00:19,697 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2902): Flushing f0e725d6c797c04eb86dfc3be87ab3ba 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-20T09:00:19,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120691c0c0ae3e148b299d6376ed6e1c534_f0e725d6c797c04eb86dfc3be87ab3ba is 50, key is test_row_0/A:col10/1732093218369/Put/seqid=0 2024-11-20T09:00:19,738 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742084_1260 (size=13255) 2024-11-20T09:00:19,753 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742085_1261 (size=12454) 2024-11-20T09:00:20,152 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/C/e54aba70aa4e49c4a3d13f361ee884ef as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/C/e54aba70aa4e49c4a3d13f361ee884ef 2024-11-20T09:00:20,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:20,163 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120691c0c0ae3e148b299d6376ed6e1c534_f0e725d6c797c04eb86dfc3be87ab3ba to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120691c0c0ae3e148b299d6376ed6e1c534_f0e725d6c797c04eb86dfc3be87ab3ba 2024-11-20T09:00:20,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/A/be0608a07ded4568963399aef08543e9, store: [table=TestAcidGuarantees family=A region=f0e725d6c797c04eb86dfc3be87ab3ba] 2024-11-20T09:00:20,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/A/be0608a07ded4568963399aef08543e9 is 175, key is test_row_0/A:col10/1732093218369/Put/seqid=0 2024-11-20T09:00:20,168 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f0e725d6c797c04eb86dfc3be87ab3ba/C of f0e725d6c797c04eb86dfc3be87ab3ba into e54aba70aa4e49c4a3d13f361ee884ef(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T09:00:20,168 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f0e725d6c797c04eb86dfc3be87ab3ba: 2024-11-20T09:00:20,169 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., storeName=f0e725d6c797c04eb86dfc3be87ab3ba/C, priority=13, startTime=1732093219606; duration=0sec 2024-11-20T09:00:20,169 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T09:00:20,169 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f0e725d6c797c04eb86dfc3be87ab3ba:C 2024-11-20T09:00:20,187 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742086_1262 (size=31255) 2024-11-20T09:00:20,188 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=423, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/A/be0608a07ded4568963399aef08543e9 2024-11-20T09:00:20,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/B/716ce311502946af9aed0ed407b2df5b is 50, key is test_row_0/B:col10/1732093218369/Put/seqid=0 2024-11-20T09:00:20,216 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742087_1263 (size=12301) 2024-11-20T09:00:20,219 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=423 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/B/716ce311502946af9aed0ed407b2df5b 2024-11-20T09:00:20,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/C/fb909aafba0149b8bbfd3d6af3f9e7fb is 50, key is test_row_0/C:col10/1732093218369/Put/seqid=0 2024-11-20T09:00:20,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742088_1264 (size=12301) 2024-11-20T09:00:20,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(8855): Flush requested on f0e725d6c797c04eb86dfc3be87ab3ba 2024-11-20T09:00:20,519 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. as already flushing 2024-11-20T09:00:20,529 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:20,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48388 deadline: 1732093280529, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:20,530 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:20,530 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:20,530 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:20,531 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:20,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48450 deadline: 1732093280530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:20,532 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:20,532 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:20,532 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:20,538 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:20,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48466 deadline: 1732093280536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:20,539 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to be1eb2620e0e,34223,1732093159960 for 'test_row_2' in f0e725d6c797c04eb86dfc3be87ab3ba of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2166 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-20T09:00:20,539 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:20,539 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:20,539 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:20,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-20T09:00:20,635 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=423 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/C/fb909aafba0149b8bbfd3d6af3f9e7fb 2024-11-20T09:00:20,639 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:20,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48388 deadline: 1732093280637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:20,640 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:20,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48450 deadline: 1732093280637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:20,640 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:20,640 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:20,640 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:20,640 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:20,640 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:20,640 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:20,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/A/be0608a07ded4568963399aef08543e9 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/A/be0608a07ded4568963399aef08543e9 2024-11-20T09:00:20,649 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/A/be0608a07ded4568963399aef08543e9, entries=150, sequenceid=423, filesize=30.5 K 2024-11-20T09:00:20,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/B/716ce311502946af9aed0ed407b2df5b as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/B/716ce311502946af9aed0ed407b2df5b 2024-11-20T09:00:20,655 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/B/716ce311502946af9aed0ed407b2df5b, entries=150, sequenceid=423, filesize=12.0 K 2024-11-20T09:00:20,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/C/fb909aafba0149b8bbfd3d6af3f9e7fb as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/C/fb909aafba0149b8bbfd3d6af3f9e7fb 2024-11-20T09:00:20,663 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/C/fb909aafba0149b8bbfd3d6af3f9e7fb, entries=150, sequenceid=423, filesize=12.0 K 2024-11-20T09:00:20,664 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for f0e725d6c797c04eb86dfc3be87ab3ba in 967ms, sequenceid=423, compaction requested=false 2024-11-20T09:00:20,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2603): Flush status journal for f0e725d6c797c04eb86dfc3be87ab3ba: 2024-11-20T09:00:20,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. 2024-11-20T09:00:20,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=51 2024-11-20T09:00:20,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.HMaster(4169): Remote procedure done, pid=51 2024-11-20T09:00:20,668 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=50 2024-11-20T09:00:20,668 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1950 sec 2024-11-20T09:00:20,670 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees in 2.2010 sec 2024-11-20T09:00:20,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(8855): Flush requested on f0e725d6c797c04eb86dfc3be87ab3ba 2024-11-20T09:00:20,849 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T09:00:20,850 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f0e725d6c797c04eb86dfc3be87ab3ba 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-20T09:00:20,861 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120268fe312876b4988a9bf034a61e2afdb_f0e725d6c797c04eb86dfc3be87ab3ba is 50, key is test_row_0/A:col10/1732093220528/Put/seqid=0 2024-11-20T09:00:20,869 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742089_1265 (size=14994) 2024-11-20T09:00:20,903 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:20,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48450 deadline: 1732093280902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:20,903 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:20,903 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:20,903 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:20,906 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:20,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48388 deadline: 1732093280903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:20,907 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:20,907 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:20,907 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:21,011 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:21,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48450 deadline: 1732093281007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:21,012 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:21,012 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:21,012 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:21,017 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:21,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48388 deadline: 1732093281017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:21,017 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:21,018 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:21,018 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:21,220 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:21,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48450 deadline: 1732093281217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:21,221 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:21,221 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:21,221 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:21,230 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:21,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48388 deadline: 1732093281226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:21,231 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:21,231 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:21,231 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:21,270 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:21,274 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120268fe312876b4988a9bf034a61e2afdb_f0e725d6c797c04eb86dfc3be87ab3ba to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120268fe312876b4988a9bf034a61e2afdb_f0e725d6c797c04eb86dfc3be87ab3ba 2024-11-20T09:00:21,275 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/A/93b5a83cf0904f7db5fdcb9102b51a9d, store: [table=TestAcidGuarantees family=A region=f0e725d6c797c04eb86dfc3be87ab3ba] 2024-11-20T09:00:21,275 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/A/93b5a83cf0904f7db5fdcb9102b51a9d is 175, key is test_row_0/A:col10/1732093220528/Put/seqid=0 2024-11-20T09:00:21,281 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742090_1266 (size=39949) 2024-11-20T09:00:21,411 INFO [Thread-715 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-715. 2024-11-20T09:00:21,411 DEBUG [Thread-715 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T09:00:21,411 DEBUG [Thread-715 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T09:00:21,411 DEBUG [Thread-715 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T09:00:21,411 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T09:00:21,415 INFO [Thread-721 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-721. 2024-11-20T09:00:21,415 DEBUG [Thread-721 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T09:00:21,415 DEBUG [Thread-721 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T09:00:21,415 INFO [Thread-717 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-717. 2024-11-20T09:00:21,415 DEBUG [Thread-721 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T09:00:21,415 DEBUG [Thread-717 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T09:00:21,415 DEBUG [Thread-717 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T09:00:21,415 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T09:00:21,415 DEBUG [Thread-717 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T09:00:21,415 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T09:00:21,420 INFO [Thread-719 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-719. 2024-11-20T09:00:21,420 DEBUG [Thread-719 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T09:00:21,420 DEBUG [Thread-719 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T09:00:21,420 DEBUG [Thread-719 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T09:00:21,420 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T09:00:21,527 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:21,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48450 deadline: 1732093281527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:21,528 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:21,528 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:21,528 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:21,537 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:21,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48388 deadline: 1732093281537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:21,537 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:21,537 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:21,538 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:21,681 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=436, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/A/93b5a83cf0904f7db5fdcb9102b51a9d 2024-11-20T09:00:21,689 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/B/b642098a6931455990f6b266051462a9 is 50, key is test_row_0/B:col10/1732093220528/Put/seqid=0 2024-11-20T09:00:21,692 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742091_1267 (size=12301) 2024-11-20T09:00:22,037 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:22,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48450 deadline: 1732093282036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:22,037 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:22,037 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:22,037 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:22,047 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:22,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48388 deadline: 1732093282047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:22,047 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:22,047 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:22,047 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:22,093 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=436 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/B/b642098a6931455990f6b266051462a9 2024-11-20T09:00:22,099 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/C/c100597108914934928df690f20ae320 is 50, key is test_row_0/C:col10/1732093220528/Put/seqid=0 2024-11-20T09:00:22,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742092_1268 (size=12301) 2024-11-20T09:00:22,503 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=436 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/C/c100597108914934928df690f20ae320 2024-11-20T09:00:22,507 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/A/93b5a83cf0904f7db5fdcb9102b51a9d as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/A/93b5a83cf0904f7db5fdcb9102b51a9d 2024-11-20T09:00:22,510 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/A/93b5a83cf0904f7db5fdcb9102b51a9d, entries=200, sequenceid=436, filesize=39.0 K 2024-11-20T09:00:22,511 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/B/b642098a6931455990f6b266051462a9 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/B/b642098a6931455990f6b266051462a9 2024-11-20T09:00:22,514 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/B/b642098a6931455990f6b266051462a9, entries=150, sequenceid=436, filesize=12.0 K 2024-11-20T09:00:22,515 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/C/c100597108914934928df690f20ae320 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/C/c100597108914934928df690f20ae320 2024-11-20T09:00:22,518 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/C/c100597108914934928df690f20ae320, entries=150, sequenceid=436, filesize=12.0 K 2024-11-20T09:00:22,519 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for f0e725d6c797c04eb86dfc3be87ab3ba in 1669ms, sequenceid=436, compaction requested=true 2024-11-20T09:00:22,519 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f0e725d6c797c04eb86dfc3be87ab3ba: 2024-11-20T09:00:22,519 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f0e725d6c797c04eb86dfc3be87ab3ba:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T09:00:22,519 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T09:00:22,519 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f0e725d6c797c04eb86dfc3be87ab3ba:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T09:00:22,519 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T09:00:22,519 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f0e725d6c797c04eb86dfc3be87ab3ba:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T09:00:22,519 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T09:00:22,519 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T09:00:22,519 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T09:00:22,520 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37891 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T09:00:22,520 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103520 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T09:00:22,520 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1541): f0e725d6c797c04eb86dfc3be87ab3ba/B is initiating minor compaction (all files) 2024-11-20T09:00:22,520 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HStore(1541): f0e725d6c797c04eb86dfc3be87ab3ba/A is initiating minor compaction (all files) 2024-11-20T09:00:22,520 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f0e725d6c797c04eb86dfc3be87ab3ba/A in TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. 2024-11-20T09:00:22,520 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f0e725d6c797c04eb86dfc3be87ab3ba/B in TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. 2024-11-20T09:00:22,520 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/B/03b3112046764f18ba8cbce9ccedc3fd, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/B/716ce311502946af9aed0ed407b2df5b, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/B/b642098a6931455990f6b266051462a9] into tmpdir=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp, totalSize=37.0 K 2024-11-20T09:00:22,520 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/A/dbc3195e2bd24ef4a99fcc8ae70e8167, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/A/be0608a07ded4568963399aef08543e9, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/A/93b5a83cf0904f7db5fdcb9102b51a9d] into tmpdir=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp, totalSize=101.1 K 2024-11-20T09:00:22,520 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. 2024-11-20T09:00:22,520 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. files: [hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/A/dbc3195e2bd24ef4a99fcc8ae70e8167, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/A/be0608a07ded4568963399aef08543e9, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/A/93b5a83cf0904f7db5fdcb9102b51a9d] 2024-11-20T09:00:22,520 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting 03b3112046764f18ba8cbce9ccedc3fd, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=395, earliestPutTs=1732093216045 2024-11-20T09:00:22,520 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.Compactor(225): Compacting dbc3195e2bd24ef4a99fcc8ae70e8167, keycount=150, bloomtype=ROW, size=31.6 K, encoding=NONE, compression=NONE, seqNum=395, earliestPutTs=1732093216045 2024-11-20T09:00:22,521 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting 716ce311502946af9aed0ed407b2df5b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=423, earliestPutTs=1732093218359 2024-11-20T09:00:22,521 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.Compactor(225): Compacting be0608a07ded4568963399aef08543e9, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=423, earliestPutTs=1732093218359 2024-11-20T09:00:22,521 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting b642098a6931455990f6b266051462a9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=436, earliestPutTs=1732093220528 2024-11-20T09:00:22,521 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.Compactor(225): Compacting 93b5a83cf0904f7db5fdcb9102b51a9d, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=436, earliestPutTs=1732093220520 2024-11-20T09:00:22,532 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f0e725d6c797c04eb86dfc3be87ab3ba#B#compaction#222 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T09:00:22,533 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/B/385e60f91a574b91bba91eecadeeadc4 is 50, key is test_row_0/B:col10/1732093220528/Put/seqid=0 2024-11-20T09:00:22,534 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=f0e725d6c797c04eb86dfc3be87ab3ba] 2024-11-20T09:00:22,537 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742093_1269 (size=13391) 2024-11-20T09:00:22,537 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411204dadf9fb22db4315958f0a70e46b6547_f0e725d6c797c04eb86dfc3be87ab3ba store=[table=TestAcidGuarantees family=A region=f0e725d6c797c04eb86dfc3be87ab3ba] 2024-11-20T09:00:22,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(8855): Flush requested on f0e725d6c797c04eb86dfc3be87ab3ba 2024-11-20T09:00:22,569 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T09:00:22,569 INFO [Thread-708 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-708. 2024-11-20T09:00:22,569 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f0e725d6c797c04eb86dfc3be87ab3ba 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-20T09:00:22,569 DEBUG [Thread-708 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T09:00:22,569 DEBUG [Thread-708 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T09:00:22,569 DEBUG [Thread-708 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T09:00:22,569 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T09:00:22,574 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411204dadf9fb22db4315958f0a70e46b6547_f0e725d6c797c04eb86dfc3be87ab3ba, store=[table=TestAcidGuarantees family=A region=f0e725d6c797c04eb86dfc3be87ab3ba] 2024-11-20T09:00:22,574 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411204dadf9fb22db4315958f0a70e46b6547_f0e725d6c797c04eb86dfc3be87ab3ba because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=f0e725d6c797c04eb86dfc3be87ab3ba] 2024-11-20T09:00:22,582 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742094_1270 (size=4469) 2024-11-20T09:00:22,583 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f0e725d6c797c04eb86dfc3be87ab3ba#A#compaction#223 average throughput is 0.50 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T09:00:22,584 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/A/7220dbaa196043d491742c6a9ab61f32 is 175, key is test_row_0/A:col10/1732093220528/Put/seqid=0 2024-11-20T09:00:22,592 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120f3d528913340442ca9258a5506b570ac_f0e725d6c797c04eb86dfc3be87ab3ba is 50, key is test_row_0/A:col10/1732093220902/Put/seqid=0 2024-11-20T09:00:22,595 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742095_1271 (size=32311) 2024-11-20T09:00:22,600 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742096_1272 (size=12454) 2024-11-20T09:00:22,601 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:22,606 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120f3d528913340442ca9258a5506b570ac_f0e725d6c797c04eb86dfc3be87ab3ba to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120f3d528913340442ca9258a5506b570ac_f0e725d6c797c04eb86dfc3be87ab3ba 2024-11-20T09:00:22,607 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/A/c77fcbd2508d4719baa7bcd006fa2657, store: [table=TestAcidGuarantees family=A region=f0e725d6c797c04eb86dfc3be87ab3ba] 2024-11-20T09:00:22,610 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/A/c77fcbd2508d4719baa7bcd006fa2657 is 175, key is test_row_0/A:col10/1732093220902/Put/seqid=0 2024-11-20T09:00:22,614 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742097_1273 (size=31255) 2024-11-20T09:00:22,615 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=461, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/A/c77fcbd2508d4719baa7bcd006fa2657 2024-11-20T09:00:22,625 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/B/38d3c2adef4a4519b8a6d1bfab6e6b99 is 50, key is test_row_0/B:col10/1732093220902/Put/seqid=0 2024-11-20T09:00:22,629 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742098_1274 (size=12301) 2024-11-20T09:00:22,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-20T09:00:22,638 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T09:00:22,944 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/B/385e60f91a574b91bba91eecadeeadc4 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/B/385e60f91a574b91bba91eecadeeadc4 2024-11-20T09:00:22,948 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f0e725d6c797c04eb86dfc3be87ab3ba/B of f0e725d6c797c04eb86dfc3be87ab3ba into 385e60f91a574b91bba91eecadeeadc4(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T09:00:22,948 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f0e725d6c797c04eb86dfc3be87ab3ba: 2024-11-20T09:00:22,948 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., storeName=f0e725d6c797c04eb86dfc3be87ab3ba/B, priority=13, startTime=1732093222519; duration=0sec 2024-11-20T09:00:22,948 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T09:00:22,948 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f0e725d6c797c04eb86dfc3be87ab3ba:B 2024-11-20T09:00:22,948 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T09:00:22,949 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37857 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T09:00:22,949 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1541): f0e725d6c797c04eb86dfc3be87ab3ba/C is initiating minor compaction (all files) 2024-11-20T09:00:22,949 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f0e725d6c797c04eb86dfc3be87ab3ba/C in TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. 2024-11-20T09:00:22,949 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/C/e54aba70aa4e49c4a3d13f361ee884ef, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/C/fb909aafba0149b8bbfd3d6af3f9e7fb, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/C/c100597108914934928df690f20ae320] into tmpdir=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp, totalSize=37.0 K 2024-11-20T09:00:22,950 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting e54aba70aa4e49c4a3d13f361ee884ef, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=395, earliestPutTs=1732093216045 2024-11-20T09:00:22,950 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting fb909aafba0149b8bbfd3d6af3f9e7fb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=423, earliestPutTs=1732093218359 2024-11-20T09:00:22,950 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting c100597108914934928df690f20ae320, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=436, earliestPutTs=1732093220528 2024-11-20T09:00:22,959 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f0e725d6c797c04eb86dfc3be87ab3ba#C#compaction#226 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T09:00:22,959 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/C/f196faf193c440f584d5640ab2c25c1f is 50, key is test_row_0/C:col10/1732093220528/Put/seqid=0 2024-11-20T09:00:22,963 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742099_1275 (size=13357) 2024-11-20T09:00:23,001 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/A/7220dbaa196043d491742c6a9ab61f32 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/A/7220dbaa196043d491742c6a9ab61f32 2024-11-20T09:00:23,006 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f0e725d6c797c04eb86dfc3be87ab3ba/A of f0e725d6c797c04eb86dfc3be87ab3ba into 7220dbaa196043d491742c6a9ab61f32(size=31.6 K), total size for store is 31.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T09:00:23,006 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f0e725d6c797c04eb86dfc3be87ab3ba: 2024-11-20T09:00:23,006 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., storeName=f0e725d6c797c04eb86dfc3be87ab3ba/A, priority=13, startTime=1732093222519; duration=0sec 2024-11-20T09:00:23,006 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T09:00:23,006 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f0e725d6c797c04eb86dfc3be87ab3ba:A 2024-11-20T09:00:23,030 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=461 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/B/38d3c2adef4a4519b8a6d1bfab6e6b99 2024-11-20T09:00:23,038 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/C/30bb23328bcf4dc6a0116d4fa02d69be is 50, key is test_row_0/C:col10/1732093220902/Put/seqid=0 2024-11-20T09:00:23,042 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742100_1276 (size=12301) 2024-11-20T09:00:23,042 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=461 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/C/30bb23328bcf4dc6a0116d4fa02d69be 2024-11-20T09:00:23,046 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/A/c77fcbd2508d4719baa7bcd006fa2657 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/A/c77fcbd2508d4719baa7bcd006fa2657 2024-11-20T09:00:23,049 INFO [Thread-710 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-710. 2024-11-20T09:00:23,049 DEBUG [Thread-710 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T09:00:23,049 DEBUG [Thread-710 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T09:00:23,049 DEBUG [Thread-710 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T09:00:23,049 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T09:00:23,050 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/A/c77fcbd2508d4719baa7bcd006fa2657, entries=150, sequenceid=461, filesize=30.5 K 2024-11-20T09:00:23,050 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/B/38d3c2adef4a4519b8a6d1bfab6e6b99 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/B/38d3c2adef4a4519b8a6d1bfab6e6b99 2024-11-20T09:00:23,054 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/B/38d3c2adef4a4519b8a6d1bfab6e6b99, entries=150, sequenceid=461, filesize=12.0 K 2024-11-20T09:00:23,054 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/C/30bb23328bcf4dc6a0116d4fa02d69be as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/C/30bb23328bcf4dc6a0116d4fa02d69be 2024-11-20T09:00:23,058 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/C/30bb23328bcf4dc6a0116d4fa02d69be, entries=150, sequenceid=461, filesize=12.0 K 2024-11-20T09:00:23,058 INFO [Thread-704 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-704. 2024-11-20T09:00:23,058 DEBUG [Thread-704 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T09:00:23,058 DEBUG [Thread-704 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T09:00:23,058 DEBUG [Thread-704 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T09:00:23,058 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T09:00:23,059 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=13.42 KB/13740 for f0e725d6c797c04eb86dfc3be87ab3ba in 489ms, sequenceid=461, compaction requested=false 2024-11-20T09:00:23,059 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f0e725d6c797c04eb86dfc3be87ab3ba: 2024-11-20T09:00:23,368 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/C/f196faf193c440f584d5640ab2c25c1f as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/C/f196faf193c440f584d5640ab2c25c1f 2024-11-20T09:00:23,373 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f0e725d6c797c04eb86dfc3be87ab3ba/C of f0e725d6c797c04eb86dfc3be87ab3ba into f196faf193c440f584d5640ab2c25c1f(size=13.0 K), total size for store is 25.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T09:00:23,373 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f0e725d6c797c04eb86dfc3be87ab3ba: 2024-11-20T09:00:23,373 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., storeName=f0e725d6c797c04eb86dfc3be87ab3ba/C, priority=13, startTime=1732093222519; duration=0sec 2024-11-20T09:00:23,373 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T09:00:23,373 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f0e725d6c797c04eb86dfc3be87ab3ba:C 2024-11-20T09:00:26,409 INFO [Thread-712 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-712. 2024-11-20T09:00:26,409 DEBUG [Thread-712 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T09:00:26,409 DEBUG [Thread-712 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T09:00:26,409 DEBUG [Thread-712 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T09:00:26,409 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T09:00:26,508 INFO [Thread-706 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-706. 2024-11-20T09:00:26,509 DEBUG [Thread-706 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T09:00:26,509 DEBUG [Thread-706 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T09:00:26,509 DEBUG [Thread-706 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T09:00:26,509 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T09:00:26,509 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-20T09:00:26,509 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 81 2024-11-20T09:00:26,509 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 53 2024-11-20T09:00:26,509 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 66 2024-11-20T09:00:26,509 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 109 2024-11-20T09:00:26,509 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 55 2024-11-20T09:00:26,509 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-20T09:00:26,509 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3260 2024-11-20T09:00:26,509 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3160 2024-11-20T09:00:26,509 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-20T09:00:26,509 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1362 2024-11-20T09:00:26,509 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4085 rows 2024-11-20T09:00:26,509 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1355 2024-11-20T09:00:26,509 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4064 rows 2024-11-20T09:00:26,509 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-20T09:00:26,509 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobMixedAtomicity(AcidGuaranteesTestBase.java:132) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-20T09:00:26,509 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T09:00:26,509 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T09:00:26,509 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T09:00:26,510 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-20T09:00:26,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] procedure2.ProcedureExecutor(1139): Stored pid=52, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-20T09:00:26,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-20T09:00:26,513 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732093226513"}]},"ts":"1732093226513"} 2024-11-20T09:00:26,514 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-20T09:00:26,514 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-20T09:00:26,515 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-20T09:00:26,516 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=54, ppid=53, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f0e725d6c797c04eb86dfc3be87ab3ba, UNASSIGN}] 2024-11-20T09:00:26,516 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=54, ppid=53, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f0e725d6c797c04eb86dfc3be87ab3ba, UNASSIGN 2024-11-20T09:00:26,517 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=54 updating hbase:meta row=f0e725d6c797c04eb86dfc3be87ab3ba, regionState=CLOSING, regionLocation=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:26,518 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=54, ppid=53, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f0e725d6c797c04eb86dfc3be87ab3ba, UNASSIGN because future has completed 2024-11-20T09:00:26,518 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-20T09:00:26,518 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE, hasLock=false; CloseRegionProcedure f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960}] 2024-11-20T09:00:26,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-20T09:00:26,670 INFO [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] handler.UnassignRegionHandler(122): Close f0e725d6c797c04eb86dfc3be87ab3ba 2024-11-20T09:00:26,670 DEBUG [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-20T09:00:26,670 DEBUG [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegion(1722): Closing f0e725d6c797c04eb86dfc3be87ab3ba, disabling compactions & flushes 2024-11-20T09:00:26,670 INFO [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. 2024-11-20T09:00:26,670 DEBUG [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. 2024-11-20T09:00:26,671 DEBUG [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. after waiting 0 ms 2024-11-20T09:00:26,671 DEBUG [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. 2024-11-20T09:00:26,671 INFO [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegion(2902): Flushing f0e725d6c797c04eb86dfc3be87ab3ba 3/3 column families, dataSize=26.84 KB heapSize=71.06 KB 2024-11-20T09:00:26,677 DEBUG [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120a53eb9484cc14706b6754408f69805d0_f0e725d6c797c04eb86dfc3be87ab3ba is 50, key is test_row_0/A:col10/1732093223057/Put/seqid=0 2024-11-20T09:00:26,680 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742101_1277 (size=12454) 2024-11-20T09:00:26,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-20T09:00:27,081 DEBUG [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:27,085 INFO [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120a53eb9484cc14706b6754408f69805d0_f0e725d6c797c04eb86dfc3be87ab3ba to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120a53eb9484cc14706b6754408f69805d0_f0e725d6c797c04eb86dfc3be87ab3ba 2024-11-20T09:00:27,086 DEBUG [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/A/a0c7099cd6ed44fa9bf22590cb0cc9f2, store: [table=TestAcidGuarantees family=A region=f0e725d6c797c04eb86dfc3be87ab3ba] 2024-11-20T09:00:27,087 DEBUG [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/A/a0c7099cd6ed44fa9bf22590cb0cc9f2 is 175, key is test_row_0/A:col10/1732093223057/Put/seqid=0 2024-11-20T09:00:27,090 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742102_1278 (size=31255) 2024-11-20T09:00:27,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-20T09:00:27,491 INFO [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=471, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/A/a0c7099cd6ed44fa9bf22590cb0cc9f2 2024-11-20T09:00:27,497 DEBUG [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/B/ea2857470d2c41d28683d4bd34887e94 is 50, key is test_row_0/B:col10/1732093223057/Put/seqid=0 2024-11-20T09:00:27,500 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742103_1279 (size=12301) 2024-11-20T09:00:27,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-20T09:00:27,901 INFO [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=471 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/B/ea2857470d2c41d28683d4bd34887e94 2024-11-20T09:00:27,907 DEBUG [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/C/2a3bb53964bb4cf88c73361779aca889 is 50, key is test_row_0/C:col10/1732093223057/Put/seqid=0 2024-11-20T09:00:27,911 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742104_1280 (size=12301) 2024-11-20T09:00:28,311 INFO [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=471 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/C/2a3bb53964bb4cf88c73361779aca889 2024-11-20T09:00:28,316 DEBUG [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/A/a0c7099cd6ed44fa9bf22590cb0cc9f2 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/A/a0c7099cd6ed44fa9bf22590cb0cc9f2 2024-11-20T09:00:28,319 INFO [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/A/a0c7099cd6ed44fa9bf22590cb0cc9f2, entries=150, sequenceid=471, filesize=30.5 K 2024-11-20T09:00:28,320 DEBUG [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/B/ea2857470d2c41d28683d4bd34887e94 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/B/ea2857470d2c41d28683d4bd34887e94 2024-11-20T09:00:28,323 INFO [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/B/ea2857470d2c41d28683d4bd34887e94, entries=150, sequenceid=471, filesize=12.0 K 2024-11-20T09:00:28,324 DEBUG [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/.tmp/C/2a3bb53964bb4cf88c73361779aca889 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/C/2a3bb53964bb4cf88c73361779aca889 2024-11-20T09:00:28,327 INFO [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/C/2a3bb53964bb4cf88c73361779aca889, entries=150, sequenceid=471, filesize=12.0 K 2024-11-20T09:00:28,328 INFO [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for f0e725d6c797c04eb86dfc3be87ab3ba in 1657ms, sequenceid=471, compaction requested=true 2024-11-20T09:00:28,329 DEBUG [StoreCloser-TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/A/66893ac4f3af4f0eb7e3d13bd68f287d, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/A/d7e57c57363942fc809ffc2f05441e30, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/A/e90cf813d9b24f4f85d4f03efc79f52e, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/A/2f744b70760844c9ac03430f5eeeedc4, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/A/6a279628dca84f46b8b6eddb02457932, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/A/ef9a68d183ea47b5a6356aa1384ee3e8, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/A/c04675b5ee5b4bd3b555cb77a691f5d1, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/A/0ff292d9febc47b896283171fd4ceb4e, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/A/1b8f2a842cc14122b7a45b0c725df0b2, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/A/d3a402d8d3604d11ade73942b38c8c00, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/A/bc898d3b742c42e79dc1af0abba0316c, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/A/be529b6eb2b54b3eae6d8c04d74ab6b1, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/A/9891fc6b68b640c7b419f1ea7c44812b, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/A/463417a27d77434b9257328958bf6bd6, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/A/9b551bc9cc5c47468c85d935334f342e, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/A/a598c6511e204fa2872c9a6bfc8e04b4, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/A/b8da9a8c4c50477d91f8dddebc8346d9, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/A/755ccc014a9e4042855401cc641be14f, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/A/7bddf792bd6c4490854387b9471a09fa, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/A/0e5829fa152e4526bda4d103be67f637, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/A/e715ea8b214146aea232c62507b45252, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/A/7421fef9056b4903b6a3c7c36fbacbb7, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/A/efd778d30df946f8a463034f813a489c, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/A/4f13e5c630ae4c438a34a6af44174dfc, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/A/fd4fd714f14047628d03b42f0ef5337b, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/A/9a4cdb9f4c4c4fe2b6ac254ea45f9ed5, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/A/9e5ea3e5bc124b69bd279d8168824cc1, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/A/188ae37a340e4059a7c4a2febbed5cc3, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/A/dbc3195e2bd24ef4a99fcc8ae70e8167, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/A/be0608a07ded4568963399aef08543e9, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/A/93b5a83cf0904f7db5fdcb9102b51a9d] to archive 2024-11-20T09:00:28,330 DEBUG [StoreCloser-TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-20T09:00:28,331 DEBUG [StoreCloser-TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/A/66893ac4f3af4f0eb7e3d13bd68f287d to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/A/66893ac4f3af4f0eb7e3d13bd68f287d 2024-11-20T09:00:28,332 DEBUG [StoreCloser-TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/A/d7e57c57363942fc809ffc2f05441e30 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/A/d7e57c57363942fc809ffc2f05441e30 2024-11-20T09:00:28,333 DEBUG [StoreCloser-TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/A/e90cf813d9b24f4f85d4f03efc79f52e to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/A/e90cf813d9b24f4f85d4f03efc79f52e 2024-11-20T09:00:28,334 DEBUG [StoreCloser-TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/A/2f744b70760844c9ac03430f5eeeedc4 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/A/2f744b70760844c9ac03430f5eeeedc4 2024-11-20T09:00:28,335 DEBUG [StoreCloser-TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/A/6a279628dca84f46b8b6eddb02457932 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/A/6a279628dca84f46b8b6eddb02457932 2024-11-20T09:00:28,336 DEBUG [StoreCloser-TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/A/ef9a68d183ea47b5a6356aa1384ee3e8 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/A/ef9a68d183ea47b5a6356aa1384ee3e8 2024-11-20T09:00:28,337 DEBUG [StoreCloser-TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/A/c04675b5ee5b4bd3b555cb77a691f5d1 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/A/c04675b5ee5b4bd3b555cb77a691f5d1 2024-11-20T09:00:28,338 DEBUG [StoreCloser-TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/A/0ff292d9febc47b896283171fd4ceb4e to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/A/0ff292d9febc47b896283171fd4ceb4e 2024-11-20T09:00:28,339 DEBUG [StoreCloser-TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/A/1b8f2a842cc14122b7a45b0c725df0b2 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/A/1b8f2a842cc14122b7a45b0c725df0b2 2024-11-20T09:00:28,340 DEBUG [StoreCloser-TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/A/d3a402d8d3604d11ade73942b38c8c00 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/A/d3a402d8d3604d11ade73942b38c8c00 2024-11-20T09:00:28,340 DEBUG [StoreCloser-TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/A/bc898d3b742c42e79dc1af0abba0316c to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/A/bc898d3b742c42e79dc1af0abba0316c 2024-11-20T09:00:28,341 DEBUG [StoreCloser-TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/A/be529b6eb2b54b3eae6d8c04d74ab6b1 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/A/be529b6eb2b54b3eae6d8c04d74ab6b1 2024-11-20T09:00:28,342 DEBUG [StoreCloser-TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/A/9891fc6b68b640c7b419f1ea7c44812b to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/A/9891fc6b68b640c7b419f1ea7c44812b 2024-11-20T09:00:28,343 DEBUG [StoreCloser-TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/A/463417a27d77434b9257328958bf6bd6 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/A/463417a27d77434b9257328958bf6bd6 2024-11-20T09:00:28,344 DEBUG [StoreCloser-TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/A/9b551bc9cc5c47468c85d935334f342e to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/A/9b551bc9cc5c47468c85d935334f342e 2024-11-20T09:00:28,345 DEBUG [StoreCloser-TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/A/a598c6511e204fa2872c9a6bfc8e04b4 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/A/a598c6511e204fa2872c9a6bfc8e04b4 2024-11-20T09:00:28,346 DEBUG [StoreCloser-TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/A/b8da9a8c4c50477d91f8dddebc8346d9 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/A/b8da9a8c4c50477d91f8dddebc8346d9 2024-11-20T09:00:28,347 DEBUG [StoreCloser-TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/A/755ccc014a9e4042855401cc641be14f to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/A/755ccc014a9e4042855401cc641be14f 2024-11-20T09:00:28,348 DEBUG [StoreCloser-TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/A/7bddf792bd6c4490854387b9471a09fa to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/A/7bddf792bd6c4490854387b9471a09fa 2024-11-20T09:00:28,349 DEBUG [StoreCloser-TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/A/0e5829fa152e4526bda4d103be67f637 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/A/0e5829fa152e4526bda4d103be67f637 2024-11-20T09:00:28,350 DEBUG [StoreCloser-TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/A/e715ea8b214146aea232c62507b45252 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/A/e715ea8b214146aea232c62507b45252 2024-11-20T09:00:28,351 DEBUG [StoreCloser-TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/A/7421fef9056b4903b6a3c7c36fbacbb7 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/A/7421fef9056b4903b6a3c7c36fbacbb7 2024-11-20T09:00:28,351 DEBUG [StoreCloser-TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/A/efd778d30df946f8a463034f813a489c to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/A/efd778d30df946f8a463034f813a489c 2024-11-20T09:00:28,352 DEBUG [StoreCloser-TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/A/4f13e5c630ae4c438a34a6af44174dfc to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/A/4f13e5c630ae4c438a34a6af44174dfc 2024-11-20T09:00:28,353 DEBUG [StoreCloser-TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/A/fd4fd714f14047628d03b42f0ef5337b to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/A/fd4fd714f14047628d03b42f0ef5337b 2024-11-20T09:00:28,354 DEBUG [StoreCloser-TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/A/9a4cdb9f4c4c4fe2b6ac254ea45f9ed5 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/A/9a4cdb9f4c4c4fe2b6ac254ea45f9ed5 2024-11-20T09:00:28,355 DEBUG [StoreCloser-TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/A/9e5ea3e5bc124b69bd279d8168824cc1 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/A/9e5ea3e5bc124b69bd279d8168824cc1 2024-11-20T09:00:28,356 DEBUG [StoreCloser-TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/A/188ae37a340e4059a7c4a2febbed5cc3 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/A/188ae37a340e4059a7c4a2febbed5cc3 2024-11-20T09:00:28,357 DEBUG [StoreCloser-TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/A/dbc3195e2bd24ef4a99fcc8ae70e8167 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/A/dbc3195e2bd24ef4a99fcc8ae70e8167 2024-11-20T09:00:28,358 DEBUG [StoreCloser-TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/A/be0608a07ded4568963399aef08543e9 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/A/be0608a07ded4568963399aef08543e9 2024-11-20T09:00:28,358 DEBUG [StoreCloser-TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/A/93b5a83cf0904f7db5fdcb9102b51a9d to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/A/93b5a83cf0904f7db5fdcb9102b51a9d 2024-11-20T09:00:28,360 DEBUG [StoreCloser-TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/B/4832cd642c74480c9642917a950df0d3, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/B/d65db5654ca24639a731ed343d9e4dfe, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/B/2dab945a9c664ebb88f30ee634af33a5, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/B/4eab81e2d5d34df589f6de8c4086c355, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/B/4ee36236777344abba8cb58c74f4f910, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/B/45abc1a01d1341d1a67cfe1881a3a929, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/B/5574eb6207f640a7bfa94653e09c9bc8, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/B/b09224912527476ca924e92bc11b8a3f, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/B/502d9dffa5284da09e81e206326a48dd, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/B/92c77a6421bf4080b952e764d72b309e, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/B/367116f84c0f41eaae074366162013f6, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/B/204bc2d3789d4ce2b16e451a8be472fa, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/B/6b8e2b4645b4449095bac273d09947bd, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/B/9a6a79bb77114833840f31896f9a97cc, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/B/28ceb0b2faf44e649b20b9718d217270, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/B/ebde470d64dc473ba2d4d3aa25cdb672, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/B/03fc26e057ef4b97b2d6ac238c33de40, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/B/cb06fa40ddbe4f9d8f4ad4982c6c01b3, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/B/7441c27cf52845cfb0214f8acfc69435, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/B/e7241f8e27094568be4bd692dccd1ee8, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/B/7757629ea3fb4086a471d6baa926888a, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/B/70fc1509b6d84c69b509de5492eabb5c, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/B/38552bf1ea2c4929b4d11bafc15b2bf7, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/B/cc5c74fdde1342f6a73c6a87e6793a50, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/B/4bfff28f225f494d8c0511ced1f271a5, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/B/bde25d9be5614b92bbac7d32df128b16, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/B/768af76c0f164e799868e867bd05a1a1, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/B/d476a20296da4aa2ab0b702cb569a05c, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/B/03b3112046764f18ba8cbce9ccedc3fd, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/B/7835282314de4adb95ec1828fcdabbef, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/B/716ce311502946af9aed0ed407b2df5b, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/B/b642098a6931455990f6b266051462a9] to archive 2024-11-20T09:00:28,361 DEBUG [StoreCloser-TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-20T09:00:28,362 DEBUG [StoreCloser-TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/B/4832cd642c74480c9642917a950df0d3 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/B/4832cd642c74480c9642917a950df0d3 2024-11-20T09:00:28,363 DEBUG [StoreCloser-TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/B/d65db5654ca24639a731ed343d9e4dfe to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/B/d65db5654ca24639a731ed343d9e4dfe 2024-11-20T09:00:28,364 DEBUG [StoreCloser-TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/B/2dab945a9c664ebb88f30ee634af33a5 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/B/2dab945a9c664ebb88f30ee634af33a5 2024-11-20T09:00:28,365 DEBUG [StoreCloser-TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/B/4eab81e2d5d34df589f6de8c4086c355 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/B/4eab81e2d5d34df589f6de8c4086c355 2024-11-20T09:00:28,366 DEBUG [StoreCloser-TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/B/4ee36236777344abba8cb58c74f4f910 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/B/4ee36236777344abba8cb58c74f4f910 2024-11-20T09:00:28,367 DEBUG [StoreCloser-TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/B/45abc1a01d1341d1a67cfe1881a3a929 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/B/45abc1a01d1341d1a67cfe1881a3a929 2024-11-20T09:00:28,367 DEBUG [StoreCloser-TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/B/5574eb6207f640a7bfa94653e09c9bc8 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/B/5574eb6207f640a7bfa94653e09c9bc8 2024-11-20T09:00:28,368 DEBUG [StoreCloser-TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/B/b09224912527476ca924e92bc11b8a3f to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/B/b09224912527476ca924e92bc11b8a3f 2024-11-20T09:00:28,369 DEBUG [StoreCloser-TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/B/502d9dffa5284da09e81e206326a48dd to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/B/502d9dffa5284da09e81e206326a48dd 2024-11-20T09:00:28,370 DEBUG [StoreCloser-TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/B/92c77a6421bf4080b952e764d72b309e to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/B/92c77a6421bf4080b952e764d72b309e 2024-11-20T09:00:28,371 DEBUG [StoreCloser-TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/B/367116f84c0f41eaae074366162013f6 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/B/367116f84c0f41eaae074366162013f6 2024-11-20T09:00:28,372 DEBUG [StoreCloser-TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/B/204bc2d3789d4ce2b16e451a8be472fa to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/B/204bc2d3789d4ce2b16e451a8be472fa 2024-11-20T09:00:28,373 DEBUG [StoreCloser-TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/B/6b8e2b4645b4449095bac273d09947bd to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/B/6b8e2b4645b4449095bac273d09947bd 2024-11-20T09:00:28,374 DEBUG [StoreCloser-TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/B/9a6a79bb77114833840f31896f9a97cc to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/B/9a6a79bb77114833840f31896f9a97cc 2024-11-20T09:00:28,375 DEBUG [StoreCloser-TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/B/28ceb0b2faf44e649b20b9718d217270 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/B/28ceb0b2faf44e649b20b9718d217270 2024-11-20T09:00:28,376 DEBUG [StoreCloser-TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/B/ebde470d64dc473ba2d4d3aa25cdb672 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/B/ebde470d64dc473ba2d4d3aa25cdb672 2024-11-20T09:00:28,377 DEBUG [StoreCloser-TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/B/03fc26e057ef4b97b2d6ac238c33de40 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/B/03fc26e057ef4b97b2d6ac238c33de40 2024-11-20T09:00:28,378 DEBUG [StoreCloser-TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/B/cb06fa40ddbe4f9d8f4ad4982c6c01b3 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/B/cb06fa40ddbe4f9d8f4ad4982c6c01b3 2024-11-20T09:00:28,378 DEBUG [StoreCloser-TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/B/7441c27cf52845cfb0214f8acfc69435 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/B/7441c27cf52845cfb0214f8acfc69435 2024-11-20T09:00:28,379 DEBUG [StoreCloser-TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/B/e7241f8e27094568be4bd692dccd1ee8 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/B/e7241f8e27094568be4bd692dccd1ee8 2024-11-20T09:00:28,380 DEBUG [StoreCloser-TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/B/7757629ea3fb4086a471d6baa926888a to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/B/7757629ea3fb4086a471d6baa926888a 2024-11-20T09:00:28,381 DEBUG [StoreCloser-TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/B/70fc1509b6d84c69b509de5492eabb5c to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/B/70fc1509b6d84c69b509de5492eabb5c 2024-11-20T09:00:28,382 DEBUG [StoreCloser-TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/B/38552bf1ea2c4929b4d11bafc15b2bf7 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/B/38552bf1ea2c4929b4d11bafc15b2bf7 2024-11-20T09:00:28,383 DEBUG [StoreCloser-TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/B/cc5c74fdde1342f6a73c6a87e6793a50 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/B/cc5c74fdde1342f6a73c6a87e6793a50 2024-11-20T09:00:28,384 DEBUG [StoreCloser-TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/B/4bfff28f225f494d8c0511ced1f271a5 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/B/4bfff28f225f494d8c0511ced1f271a5 2024-11-20T09:00:28,385 DEBUG [StoreCloser-TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/B/bde25d9be5614b92bbac7d32df128b16 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/B/bde25d9be5614b92bbac7d32df128b16 2024-11-20T09:00:28,386 DEBUG [StoreCloser-TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/B/768af76c0f164e799868e867bd05a1a1 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/B/768af76c0f164e799868e867bd05a1a1 2024-11-20T09:00:28,387 DEBUG [StoreCloser-TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/B/d476a20296da4aa2ab0b702cb569a05c to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/B/d476a20296da4aa2ab0b702cb569a05c 2024-11-20T09:00:28,388 DEBUG [StoreCloser-TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/B/03b3112046764f18ba8cbce9ccedc3fd to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/B/03b3112046764f18ba8cbce9ccedc3fd 2024-11-20T09:00:28,389 DEBUG [StoreCloser-TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/B/7835282314de4adb95ec1828fcdabbef to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/B/7835282314de4adb95ec1828fcdabbef 2024-11-20T09:00:28,390 DEBUG [StoreCloser-TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/B/716ce311502946af9aed0ed407b2df5b to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/B/716ce311502946af9aed0ed407b2df5b 2024-11-20T09:00:28,391 DEBUG [StoreCloser-TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/B/b642098a6931455990f6b266051462a9 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/B/b642098a6931455990f6b266051462a9 2024-11-20T09:00:28,392 DEBUG [StoreCloser-TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/C/3e6de594cc9e4503a7378c8133a5ef86, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/C/7bd85b04e36c47d89ce8b634e3336d35, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/C/3126051b8ac2415593113cc36e85695f, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/C/2b6dcdb6782543a0aaf59181e6b850ac, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/C/0377e3b27ed74c2b8aa8a87dd9121431, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/C/04fb40e09f9f4dfca658cfc049179d49, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/C/9d0779fadc434aaf868ac25f9fcb945a, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/C/7cad10a70d7b402381ce9c9baccd6e65, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/C/8bb08d6d9e324cf0b49265c8b5e62257, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/C/1190d219706e44279cb3f77e116de28b, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/C/b5ea089291a74d1ba31b13d0ce3c7b92, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/C/e74622048252400ebdaf91b56348ac98, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/C/bcf5b04ce6fc4895a880cb7852a4a3dc, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/C/4aaf4296b433406c992433b14fb73d34, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/C/1f55295c3ec0486cb108bddf3343dcbc, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/C/d1258e24721f4ea28fa6866bd79c43c2, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/C/5a50a19faa154f1eabb1df95b5931a65, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/C/27dbbdc0cebd4201ad0eb2d133f4f2e9, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/C/1a5f1d8050704d5a817a49219f255199, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/C/dc4c8f65ee594a12b6545f4d9667bbff, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/C/8caf0f987d6b4226bd4f4f19cd0fdedd, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/C/74d11fcdc21f4e02abae769183a1cbe2, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/C/030d34f5cdcd4e64817eb41c16b5d36c, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/C/a89d18a40d674af9b7d7bf1ae5a9dc98, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/C/b69bfcbd2db842a7b1f5468d6e073679, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/C/decb431db6644214bc25ed01c7cd1fd0, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/C/05a47f98435046c696309bb92ba289d1, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/C/e54aba70aa4e49c4a3d13f361ee884ef, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/C/db43dd2d310a4435a7b01da49c075af6, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/C/fb909aafba0149b8bbfd3d6af3f9e7fb, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/C/c100597108914934928df690f20ae320] to archive 2024-11-20T09:00:28,393 DEBUG [StoreCloser-TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-20T09:00:28,394 DEBUG [StoreCloser-TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/C/3e6de594cc9e4503a7378c8133a5ef86 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/C/3e6de594cc9e4503a7378c8133a5ef86 2024-11-20T09:00:28,395 DEBUG [StoreCloser-TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/C/7bd85b04e36c47d89ce8b634e3336d35 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/C/7bd85b04e36c47d89ce8b634e3336d35 2024-11-20T09:00:28,396 DEBUG [StoreCloser-TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/C/3126051b8ac2415593113cc36e85695f to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/C/3126051b8ac2415593113cc36e85695f 2024-11-20T09:00:28,397 DEBUG [StoreCloser-TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/C/2b6dcdb6782543a0aaf59181e6b850ac to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/C/2b6dcdb6782543a0aaf59181e6b850ac 2024-11-20T09:00:28,398 DEBUG [StoreCloser-TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/C/0377e3b27ed74c2b8aa8a87dd9121431 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/C/0377e3b27ed74c2b8aa8a87dd9121431 2024-11-20T09:00:28,399 DEBUG [StoreCloser-TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/C/04fb40e09f9f4dfca658cfc049179d49 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/C/04fb40e09f9f4dfca658cfc049179d49 2024-11-20T09:00:28,400 DEBUG [StoreCloser-TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/C/9d0779fadc434aaf868ac25f9fcb945a to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/C/9d0779fadc434aaf868ac25f9fcb945a 2024-11-20T09:00:28,401 DEBUG [StoreCloser-TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/C/7cad10a70d7b402381ce9c9baccd6e65 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/C/7cad10a70d7b402381ce9c9baccd6e65 2024-11-20T09:00:28,402 DEBUG [StoreCloser-TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/C/8bb08d6d9e324cf0b49265c8b5e62257 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/C/8bb08d6d9e324cf0b49265c8b5e62257 2024-11-20T09:00:28,403 DEBUG [StoreCloser-TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/C/1190d219706e44279cb3f77e116de28b to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/C/1190d219706e44279cb3f77e116de28b 2024-11-20T09:00:28,404 DEBUG [StoreCloser-TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/C/b5ea089291a74d1ba31b13d0ce3c7b92 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/C/b5ea089291a74d1ba31b13d0ce3c7b92 2024-11-20T09:00:28,405 DEBUG [StoreCloser-TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/C/e74622048252400ebdaf91b56348ac98 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/C/e74622048252400ebdaf91b56348ac98 2024-11-20T09:00:28,406 DEBUG [StoreCloser-TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/C/bcf5b04ce6fc4895a880cb7852a4a3dc to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/C/bcf5b04ce6fc4895a880cb7852a4a3dc 2024-11-20T09:00:28,407 DEBUG [StoreCloser-TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/C/4aaf4296b433406c992433b14fb73d34 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/C/4aaf4296b433406c992433b14fb73d34 2024-11-20T09:00:28,408 DEBUG [StoreCloser-TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/C/1f55295c3ec0486cb108bddf3343dcbc to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/C/1f55295c3ec0486cb108bddf3343dcbc 2024-11-20T09:00:28,409 DEBUG [StoreCloser-TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/C/d1258e24721f4ea28fa6866bd79c43c2 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/C/d1258e24721f4ea28fa6866bd79c43c2 2024-11-20T09:00:28,410 DEBUG [StoreCloser-TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/C/5a50a19faa154f1eabb1df95b5931a65 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/C/5a50a19faa154f1eabb1df95b5931a65 2024-11-20T09:00:28,411 DEBUG [StoreCloser-TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/C/27dbbdc0cebd4201ad0eb2d133f4f2e9 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/C/27dbbdc0cebd4201ad0eb2d133f4f2e9 2024-11-20T09:00:28,412 DEBUG [StoreCloser-TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/C/1a5f1d8050704d5a817a49219f255199 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/C/1a5f1d8050704d5a817a49219f255199 2024-11-20T09:00:28,413 DEBUG [StoreCloser-TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/C/dc4c8f65ee594a12b6545f4d9667bbff to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/C/dc4c8f65ee594a12b6545f4d9667bbff 2024-11-20T09:00:28,414 DEBUG [StoreCloser-TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/C/8caf0f987d6b4226bd4f4f19cd0fdedd to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/C/8caf0f987d6b4226bd4f4f19cd0fdedd 2024-11-20T09:00:28,415 DEBUG [StoreCloser-TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/C/74d11fcdc21f4e02abae769183a1cbe2 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/C/74d11fcdc21f4e02abae769183a1cbe2 2024-11-20T09:00:28,415 DEBUG [StoreCloser-TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/C/030d34f5cdcd4e64817eb41c16b5d36c to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/C/030d34f5cdcd4e64817eb41c16b5d36c 2024-11-20T09:00:28,416 DEBUG [StoreCloser-TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/C/a89d18a40d674af9b7d7bf1ae5a9dc98 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/C/a89d18a40d674af9b7d7bf1ae5a9dc98 2024-11-20T09:00:28,417 DEBUG [StoreCloser-TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/C/b69bfcbd2db842a7b1f5468d6e073679 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/C/b69bfcbd2db842a7b1f5468d6e073679 2024-11-20T09:00:28,418 DEBUG [StoreCloser-TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/C/decb431db6644214bc25ed01c7cd1fd0 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/C/decb431db6644214bc25ed01c7cd1fd0 2024-11-20T09:00:28,419 DEBUG [StoreCloser-TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/C/05a47f98435046c696309bb92ba289d1 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/C/05a47f98435046c696309bb92ba289d1 2024-11-20T09:00:28,420 DEBUG [StoreCloser-TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/C/e54aba70aa4e49c4a3d13f361ee884ef to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/C/e54aba70aa4e49c4a3d13f361ee884ef 2024-11-20T09:00:28,421 DEBUG [StoreCloser-TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/C/db43dd2d310a4435a7b01da49c075af6 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/C/db43dd2d310a4435a7b01da49c075af6 2024-11-20T09:00:28,422 DEBUG [StoreCloser-TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/C/fb909aafba0149b8bbfd3d6af3f9e7fb to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/C/fb909aafba0149b8bbfd3d6af3f9e7fb 2024-11-20T09:00:28,422 DEBUG [StoreCloser-TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/C/c100597108914934928df690f20ae320 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/C/c100597108914934928df690f20ae320 2024-11-20T09:00:28,427 DEBUG [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/recovered.edits/474.seqid, newMaxSeqId=474, maxSeqId=4 2024-11-20T09:00:28,428 INFO [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. 2024-11-20T09:00:28,428 DEBUG [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegion(1676): Region close journal for f0e725d6c797c04eb86dfc3be87ab3ba: Waiting for close lock at 1732093226670Running coprocessor pre-close hooks at 1732093226670Disabling compacts and flushes for region at 1732093226670Disabling writes for close at 1732093226671 (+1 ms)Obtaining lock to block concurrent updates at 1732093226671Preparing flush snapshotting stores in f0e725d6c797c04eb86dfc3be87ab3ba at 1732093226671Finished memstore snapshotting TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=72720, getOffHeapSize=0, getCellsCount=600 at 1732093226671Flushing stores of TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba. at 1732093226671Flushing f0e725d6c797c04eb86dfc3be87ab3ba/A: creating writer at 1732093226672 (+1 ms)Flushing mob file f0e725d6c797c04eb86dfc3be87ab3ba/A: appending metadata at 1732093226676 (+4 ms)Flushing mob file f0e725d6c797c04eb86dfc3be87ab3ba/A: closing flushed file at 1732093226676Flushing f0e725d6c797c04eb86dfc3be87ab3ba/A: appending metadata at 1732093227086 (+410 ms)Flushing f0e725d6c797c04eb86dfc3be87ab3ba/A: closing flushed file at 1732093227086Flushing f0e725d6c797c04eb86dfc3be87ab3ba/B: creating writer at 1732093227494 (+408 ms)Flushing f0e725d6c797c04eb86dfc3be87ab3ba/B: appending metadata at 1732093227497 (+3 ms)Flushing f0e725d6c797c04eb86dfc3be87ab3ba/B: closing flushed file at 1732093227497Flushing f0e725d6c797c04eb86dfc3be87ab3ba/C: creating writer at 1732093227905 (+408 ms)Flushing f0e725d6c797c04eb86dfc3be87ab3ba/C: appending metadata at 1732093227907 (+2 ms)Flushing f0e725d6c797c04eb86dfc3be87ab3ba/C: closing flushed file at 1732093227907Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4a0b7c38: reopening flushed file at 1732093228315 (+408 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@51e26cbb: reopening flushed file at 1732093228319 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1b1c1fd4: reopening flushed file at 1732093228324 (+5 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for f0e725d6c797c04eb86dfc3be87ab3ba in 1657ms, sequenceid=471, compaction requested=true at 1732093228328 (+4 ms)Writing region close event to WAL at 1732093228424 (+96 ms)Running coprocessor post-close hooks at 1732093228428 (+4 ms)Closed at 1732093228428 2024-11-20T09:00:28,430 INFO [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] handler.UnassignRegionHandler(157): Closed f0e725d6c797c04eb86dfc3be87ab3ba 2024-11-20T09:00:28,430 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=54 updating hbase:meta row=f0e725d6c797c04eb86dfc3be87ab3ba, regionState=CLOSED 2024-11-20T09:00:28,432 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=55, ppid=54, state=RUNNABLE, hasLock=false; CloseRegionProcedure f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 because future has completed 2024-11-20T09:00:28,434 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=54 2024-11-20T09:00:28,434 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=54, state=SUCCESS, hasLock=false; CloseRegionProcedure f0e725d6c797c04eb86dfc3be87ab3ba, server=be1eb2620e0e,34223,1732093159960 in 1.9150 sec 2024-11-20T09:00:28,436 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=54, resume processing ppid=53 2024-11-20T09:00:28,436 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, ppid=53, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f0e725d6c797c04eb86dfc3be87ab3ba, UNASSIGN in 1.9180 sec 2024-11-20T09:00:28,438 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=52 2024-11-20T09:00:28,438 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=52, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9210 sec 2024-11-20T09:00:28,439 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732093228439"}]},"ts":"1732093228439"} 2024-11-20T09:00:28,441 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-20T09:00:28,441 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-20T09:00:28,442 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9310 sec 2024-11-20T09:00:28,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-20T09:00:28,657 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-20T09:00:28,657 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-20T09:00:28,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] procedure2.ProcedureExecutor(1139): Stored pid=56, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T09:00:28,659 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=56, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T09:00:28,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-20T09:00:28,660 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=56, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T09:00:28,663 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba 2024-11-20T09:00:28,665 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/A, FileablePath, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/B, FileablePath, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/C, FileablePath, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/recovered.edits] 2024-11-20T09:00:28,668 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/A/7220dbaa196043d491742c6a9ab61f32 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/A/7220dbaa196043d491742c6a9ab61f32 2024-11-20T09:00:28,669 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/A/a0c7099cd6ed44fa9bf22590cb0cc9f2 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/A/a0c7099cd6ed44fa9bf22590cb0cc9f2 2024-11-20T09:00:28,670 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/A/c77fcbd2508d4719baa7bcd006fa2657 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/A/c77fcbd2508d4719baa7bcd006fa2657 2024-11-20T09:00:28,672 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/B/385e60f91a574b91bba91eecadeeadc4 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/B/385e60f91a574b91bba91eecadeeadc4 2024-11-20T09:00:28,673 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/B/38d3c2adef4a4519b8a6d1bfab6e6b99 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/B/38d3c2adef4a4519b8a6d1bfab6e6b99 2024-11-20T09:00:28,674 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/B/ea2857470d2c41d28683d4bd34887e94 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/B/ea2857470d2c41d28683d4bd34887e94 2024-11-20T09:00:28,676 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/C/2a3bb53964bb4cf88c73361779aca889 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/C/2a3bb53964bb4cf88c73361779aca889 2024-11-20T09:00:28,677 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/C/30bb23328bcf4dc6a0116d4fa02d69be to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/C/30bb23328bcf4dc6a0116d4fa02d69be 2024-11-20T09:00:28,678 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/C/f196faf193c440f584d5640ab2c25c1f to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/C/f196faf193c440f584d5640ab2c25c1f 2024-11-20T09:00:28,680 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/recovered.edits/474.seqid to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba/recovered.edits/474.seqid 2024-11-20T09:00:28,681 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/f0e725d6c797c04eb86dfc3be87ab3ba 2024-11-20T09:00:28,681 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-20T09:00:28,681 DEBUG [PEWorker-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-20T09:00:28,682 DEBUG [PEWorker-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-20T09:00:28,685 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411200baa6142cbf34b07a8f2cf11953340a3_f0e725d6c797c04eb86dfc3be87ab3ba to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411200baa6142cbf34b07a8f2cf11953340a3_f0e725d6c797c04eb86dfc3be87ab3ba 2024-11-20T09:00:28,686 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411200f3103f23afa478ab84384d04d1bbf7c_f0e725d6c797c04eb86dfc3be87ab3ba to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411200f3103f23afa478ab84384d04d1bbf7c_f0e725d6c797c04eb86dfc3be87ab3ba 2024-11-20T09:00:28,687 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112013a4bed7123e4ab0829eefdb9f90dc6c_f0e725d6c797c04eb86dfc3be87ab3ba to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112013a4bed7123e4ab0829eefdb9f90dc6c_f0e725d6c797c04eb86dfc3be87ab3ba 2024-11-20T09:00:28,688 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120268fe312876b4988a9bf034a61e2afdb_f0e725d6c797c04eb86dfc3be87ab3ba to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120268fe312876b4988a9bf034a61e2afdb_f0e725d6c797c04eb86dfc3be87ab3ba 2024-11-20T09:00:28,689 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112041f5b8801dc94e0f96fd85a022b02161_f0e725d6c797c04eb86dfc3be87ab3ba to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112041f5b8801dc94e0f96fd85a022b02161_f0e725d6c797c04eb86dfc3be87ab3ba 2024-11-20T09:00:28,690 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112059af40569c5444eda0e0f6bf6a9a1b0b_f0e725d6c797c04eb86dfc3be87ab3ba to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112059af40569c5444eda0e0f6bf6a9a1b0b_f0e725d6c797c04eb86dfc3be87ab3ba 2024-11-20T09:00:28,691 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411205d3df700967f4b91bc0f6bd9ac1314cd_f0e725d6c797c04eb86dfc3be87ab3ba to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411205d3df700967f4b91bc0f6bd9ac1314cd_f0e725d6c797c04eb86dfc3be87ab3ba 2024-11-20T09:00:28,692 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120614a9b6824b5479986171cd202c1cda0_f0e725d6c797c04eb86dfc3be87ab3ba to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120614a9b6824b5479986171cd202c1cda0_f0e725d6c797c04eb86dfc3be87ab3ba 2024-11-20T09:00:28,693 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120691c0c0ae3e148b299d6376ed6e1c534_f0e725d6c797c04eb86dfc3be87ab3ba to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120691c0c0ae3e148b299d6376ed6e1c534_f0e725d6c797c04eb86dfc3be87ab3ba 2024-11-20T09:00:28,694 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112073d03e8b9ba34ee39175adb74db7db5f_f0e725d6c797c04eb86dfc3be87ab3ba to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112073d03e8b9ba34ee39175adb74db7db5f_f0e725d6c797c04eb86dfc3be87ab3ba 2024-11-20T09:00:28,695 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120799c0e309fac4ab3954500cbc6b5b52e_f0e725d6c797c04eb86dfc3be87ab3ba to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120799c0e309fac4ab3954500cbc6b5b52e_f0e725d6c797c04eb86dfc3be87ab3ba 2024-11-20T09:00:28,696 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411208422843b2dcd49caa696d9fbfbb50afe_f0e725d6c797c04eb86dfc3be87ab3ba to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411208422843b2dcd49caa696d9fbfbb50afe_f0e725d6c797c04eb86dfc3be87ab3ba 2024-11-20T09:00:28,697 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112086509b49a07849dba8dfb5f9bf418f94_f0e725d6c797c04eb86dfc3be87ab3ba to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112086509b49a07849dba8dfb5f9bf418f94_f0e725d6c797c04eb86dfc3be87ab3ba 2024-11-20T09:00:28,698 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411208820fb202895484fb7ed9f4f02ea40ff_f0e725d6c797c04eb86dfc3be87ab3ba to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411208820fb202895484fb7ed9f4f02ea40ff_f0e725d6c797c04eb86dfc3be87ab3ba 2024-11-20T09:00:28,699 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120890d266f803c4681aaf1ffe741598916_f0e725d6c797c04eb86dfc3be87ab3ba to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120890d266f803c4681aaf1ffe741598916_f0e725d6c797c04eb86dfc3be87ab3ba 2024-11-20T09:00:28,700 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411209856c6312834472b8e45d3b103637f94_f0e725d6c797c04eb86dfc3be87ab3ba to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411209856c6312834472b8e45d3b103637f94_f0e725d6c797c04eb86dfc3be87ab3ba 2024-11-20T09:00:28,701 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411209aca9bdb2ebe4142977109b6a8305496_f0e725d6c797c04eb86dfc3be87ab3ba to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411209aca9bdb2ebe4142977109b6a8305496_f0e725d6c797c04eb86dfc3be87ab3ba 2024-11-20T09:00:28,702 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411209ebd9c801ef2461592da50bb6b464429_f0e725d6c797c04eb86dfc3be87ab3ba to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411209ebd9c801ef2461592da50bb6b464429_f0e725d6c797c04eb86dfc3be87ab3ba 2024-11-20T09:00:28,703 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120a53eb9484cc14706b6754408f69805d0_f0e725d6c797c04eb86dfc3be87ab3ba to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120a53eb9484cc14706b6754408f69805d0_f0e725d6c797c04eb86dfc3be87ab3ba 2024-11-20T09:00:28,703 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120a8fcf24fea8147a299cfedafc2529b6d_f0e725d6c797c04eb86dfc3be87ab3ba to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120a8fcf24fea8147a299cfedafc2529b6d_f0e725d6c797c04eb86dfc3be87ab3ba 2024-11-20T09:00:28,705 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120d1a99ca3505d495cadf1ecda38802808_f0e725d6c797c04eb86dfc3be87ab3ba to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120d1a99ca3505d495cadf1ecda38802808_f0e725d6c797c04eb86dfc3be87ab3ba 2024-11-20T09:00:28,706 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120e94abf5d9d5f4388beb7d4cafcd74124_f0e725d6c797c04eb86dfc3be87ab3ba to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120e94abf5d9d5f4388beb7d4cafcd74124_f0e725d6c797c04eb86dfc3be87ab3ba 2024-11-20T09:00:28,708 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120ee026e55f2134e468f29a582466bda6b_f0e725d6c797c04eb86dfc3be87ab3ba to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120ee026e55f2134e468f29a582466bda6b_f0e725d6c797c04eb86dfc3be87ab3ba 2024-11-20T09:00:28,709 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120f11506a113fa40849aaadb26f853ed6f_f0e725d6c797c04eb86dfc3be87ab3ba to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120f11506a113fa40849aaadb26f853ed6f_f0e725d6c797c04eb86dfc3be87ab3ba 2024-11-20T09:00:28,710 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120f3d528913340442ca9258a5506b570ac_f0e725d6c797c04eb86dfc3be87ab3ba to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120f3d528913340442ca9258a5506b570ac_f0e725d6c797c04eb86dfc3be87ab3ba 2024-11-20T09:00:28,710 DEBUG [PEWorker-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-20T09:00:28,712 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=56, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T09:00:28,716 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-20T09:00:28,718 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-20T09:00:28,719 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=56, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T09:00:28,719 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-20T09:00:28,719 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732093228719"}]},"ts":"9223372036854775807"} 2024-11-20T09:00:28,721 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-20T09:00:28,721 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => f0e725d6c797c04eb86dfc3be87ab3ba, NAME => 'TestAcidGuarantees,,1732093198255.f0e725d6c797c04eb86dfc3be87ab3ba.', STARTKEY => '', ENDKEY => ''}] 2024-11-20T09:00:28,721 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-20T09:00:28,721 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732093228721"}]},"ts":"9223372036854775807"} 2024-11-20T09:00:28,723 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-20T09:00:28,724 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=56, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T09:00:28,725 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 67 msec 2024-11-20T09:00:28,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-20T09:00:28,767 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-20T09:00:28,767 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-20T09:00:28,776 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobMixedAtomicity Thread=239 (was 236) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d52d975-8094-4c70-a6d2-7d2f656df145/cluster_0d88e112-ef41-ee03-4061-1e732edd46d6/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_656394272_22 at /127.0.0.1:38046 [Waiting for operation #15] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d52d975-8094-4c70-a6d2-7d2f656df145/cluster_0d88e112-ef41-ee03-4061-1e732edd46d6/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=455 (was 449) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=486 (was 381) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=2143 (was 2372) 2024-11-20T09:00:28,784 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testGetAtomicity Thread=239, OpenFileDescriptor=455, MaxFileDescriptor=1048576, SystemLoadAverage=486, ProcessCount=11, AvailableMemoryMB=2143 2024-11-20T09:00:28,786 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-20T09:00:28,786 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-20T09:00:28,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] procedure2.ProcedureExecutor(1139): Stored pid=57, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-20T09:00:28,788 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=57, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-20T09:00:28,788 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:28,788 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 57 2024-11-20T09:00:28,789 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=57, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-20T09:00:28,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=57 2024-11-20T09:00:28,794 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742105_1281 (size=959) 2024-11-20T09:00:28,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=57 2024-11-20T09:00:29,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=57 2024-11-20T09:00:29,196 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 11f6fecb00005c6636e7ea541ff70dfb, NAME => 'TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a 2024-11-20T09:00:29,201 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742106_1282 (size=53) 2024-11-20T09:00:29,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=57 2024-11-20T09:00:29,602 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T09:00:29,602 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 11f6fecb00005c6636e7ea541ff70dfb, disabling compactions & flushes 2024-11-20T09:00:29,602 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb. 2024-11-20T09:00:29,602 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb. 2024-11-20T09:00:29,602 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb. after waiting 0 ms 2024-11-20T09:00:29,602 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb. 2024-11-20T09:00:29,602 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb. 2024-11-20T09:00:29,602 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 11f6fecb00005c6636e7ea541ff70dfb: Waiting for close lock at 1732093229602Disabling compacts and flushes for region at 1732093229602Disabling writes for close at 1732093229602Writing region close event to WAL at 1732093229602Closed at 1732093229602 2024-11-20T09:00:29,604 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=57, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-20T09:00:29,604 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732093229604"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732093229604"}]},"ts":"1732093229604"} 2024-11-20T09:00:29,606 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-20T09:00:29,606 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=57, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-20T09:00:29,607 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732093229606"}]},"ts":"1732093229606"} 2024-11-20T09:00:29,608 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-20T09:00:29,608 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=11f6fecb00005c6636e7ea541ff70dfb, ASSIGN}] 2024-11-20T09:00:29,609 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=11f6fecb00005c6636e7ea541ff70dfb, ASSIGN 2024-11-20T09:00:29,610 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=11f6fecb00005c6636e7ea541ff70dfb, ASSIGN; state=OFFLINE, location=be1eb2620e0e,34223,1732093159960; forceNewPlan=false, retain=false 2024-11-20T09:00:29,645 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-20T09:00:29,761 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=58 updating hbase:meta row=11f6fecb00005c6636e7ea541ff70dfb, regionState=OPENING, regionLocation=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:29,763 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=11f6fecb00005c6636e7ea541ff70dfb, ASSIGN because future has completed 2024-11-20T09:00:29,763 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE, hasLock=false; OpenRegionProcedure 11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960}] 2024-11-20T09:00:29,918 INFO [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb. 2024-11-20T09:00:29,918 DEBUG [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(7752): Opening region: {ENCODED => 11f6fecb00005c6636e7ea541ff70dfb, NAME => 'TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.', STARTKEY => '', ENDKEY => ''} 2024-11-20T09:00:29,918 DEBUG [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 11f6fecb00005c6636e7ea541ff70dfb 2024-11-20T09:00:29,918 DEBUG [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T09:00:29,918 DEBUG [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(7794): checking encryption for 11f6fecb00005c6636e7ea541ff70dfb 2024-11-20T09:00:29,918 DEBUG [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(7797): checking classloading for 11f6fecb00005c6636e7ea541ff70dfb 2024-11-20T09:00:29,919 INFO [StoreOpener-11f6fecb00005c6636e7ea541ff70dfb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 11f6fecb00005c6636e7ea541ff70dfb 2024-11-20T09:00:29,921 INFO [StoreOpener-11f6fecb00005c6636e7ea541ff70dfb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 11f6fecb00005c6636e7ea541ff70dfb columnFamilyName A 2024-11-20T09:00:29,921 DEBUG [StoreOpener-11f6fecb00005c6636e7ea541ff70dfb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:29,921 INFO [StoreOpener-11f6fecb00005c6636e7ea541ff70dfb-1 {}] regionserver.HStore(327): Store=11f6fecb00005c6636e7ea541ff70dfb/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T09:00:29,921 INFO [StoreOpener-11f6fecb00005c6636e7ea541ff70dfb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 11f6fecb00005c6636e7ea541ff70dfb 2024-11-20T09:00:29,922 INFO [StoreOpener-11f6fecb00005c6636e7ea541ff70dfb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 11f6fecb00005c6636e7ea541ff70dfb columnFamilyName B 2024-11-20T09:00:29,922 DEBUG [StoreOpener-11f6fecb00005c6636e7ea541ff70dfb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:29,923 INFO [StoreOpener-11f6fecb00005c6636e7ea541ff70dfb-1 {}] regionserver.HStore(327): Store=11f6fecb00005c6636e7ea541ff70dfb/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T09:00:29,923 INFO [StoreOpener-11f6fecb00005c6636e7ea541ff70dfb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 11f6fecb00005c6636e7ea541ff70dfb 2024-11-20T09:00:29,924 INFO [StoreOpener-11f6fecb00005c6636e7ea541ff70dfb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 11f6fecb00005c6636e7ea541ff70dfb columnFamilyName C 2024-11-20T09:00:29,924 DEBUG [StoreOpener-11f6fecb00005c6636e7ea541ff70dfb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:29,924 INFO [StoreOpener-11f6fecb00005c6636e7ea541ff70dfb-1 {}] regionserver.HStore(327): Store=11f6fecb00005c6636e7ea541ff70dfb/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T09:00:29,924 DEBUG [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1038): replaying wal for 11f6fecb00005c6636e7ea541ff70dfb 2024-11-20T09:00:29,925 DEBUG [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb 2024-11-20T09:00:29,925 DEBUG [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb 2024-11-20T09:00:29,926 DEBUG [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1048): stopping wal replay for 11f6fecb00005c6636e7ea541ff70dfb 2024-11-20T09:00:29,926 DEBUG [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1060): Cleaning up temporary data for 11f6fecb00005c6636e7ea541ff70dfb 2024-11-20T09:00:29,926 DEBUG [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-20T09:00:29,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=57 2024-11-20T09:00:29,927 DEBUG [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1093): writing seq id for 11f6fecb00005c6636e7ea541ff70dfb 2024-11-20T09:00:29,929 DEBUG [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T09:00:29,929 INFO [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1114): Opened 11f6fecb00005c6636e7ea541ff70dfb; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72137545, jitterRate=0.07493318617343903}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-20T09:00:29,929 DEBUG [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 11f6fecb00005c6636e7ea541ff70dfb 2024-11-20T09:00:29,930 DEBUG [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1006): Region open journal for 11f6fecb00005c6636e7ea541ff70dfb: Running coprocessor pre-open hook at 1732093229918Writing region info on filesystem at 1732093229918Initializing all the Stores at 1732093229919 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732093229919Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732093229919Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732093229919Cleaning up temporary data from old regions at 1732093229926 (+7 ms)Running coprocessor post-open hooks at 1732093229929 (+3 ms)Region opened successfully at 1732093229930 (+1 ms) 2024-11-20T09:00:29,930 INFO [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., pid=59, masterSystemTime=1732093229915 2024-11-20T09:00:29,932 DEBUG [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb. 2024-11-20T09:00:29,932 INFO [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb. 2024-11-20T09:00:29,933 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=58 updating hbase:meta row=11f6fecb00005c6636e7ea541ff70dfb, regionState=OPEN, openSeqNum=2, regionLocation=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:29,934 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=59, ppid=58, state=RUNNABLE, hasLock=false; OpenRegionProcedure 11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 because future has completed 2024-11-20T09:00:29,936 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=58 2024-11-20T09:00:29,936 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=58, state=SUCCESS, hasLock=false; OpenRegionProcedure 11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 in 172 msec 2024-11-20T09:00:29,937 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=58, resume processing ppid=57 2024-11-20T09:00:29,938 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=11f6fecb00005c6636e7ea541ff70dfb, ASSIGN in 328 msec 2024-11-20T09:00:29,938 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=57, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-20T09:00:29,939 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732093229938"}]},"ts":"1732093229938"} 2024-11-20T09:00:29,940 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-20T09:00:29,940 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=57, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-20T09:00:29,941 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1540 sec 2024-11-20T09:00:30,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=57 2024-11-20T09:00:30,937 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-20T09:00:30,938 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@555e4e20, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T09:00:30,938 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request be1eb2620e0e,44373,-1 for getting cluster id 2024-11-20T09:00:30,938 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T09:00:30,939 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'caf50035-b19e-4cfd-90ea-dca559866882' 2024-11-20T09:00:30,940 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T09:00:30,940 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "caf50035-b19e-4cfd-90ea-dca559866882" 2024-11-20T09:00:30,940 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@486026ef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T09:00:30,940 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [be1eb2620e0e,44373,-1] 2024-11-20T09:00:30,940 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T09:00:30,940 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T09:00:30,941 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52774, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T09:00:30,942 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@30b6803e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T09:00:30,942 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T09:00:30,943 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=be1eb2620e0e,34223,1732093159960, seqNum=-1] 2024-11-20T09:00:30,943 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T09:00:30,944 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58442, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T09:00:30,944 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-20T09:00:30,945 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncConnectionImpl(321): The fetched master address is be1eb2620e0e,44373,1732093159150 2024-11-20T09:00:30,945 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@28398524 2024-11-20T09:00:30,945 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-20T09:00:30,946 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52778, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-20T09:00:30,947 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4cf99f0a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T09:00:30,947 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request be1eb2620e0e,44373,-1 for getting cluster id 2024-11-20T09:00:30,947 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T09:00:30,947 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'caf50035-b19e-4cfd-90ea-dca559866882' 2024-11-20T09:00:30,947 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T09:00:30,948 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "caf50035-b19e-4cfd-90ea-dca559866882" 2024-11-20T09:00:30,948 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@370bb271, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T09:00:30,948 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [be1eb2620e0e,44373,-1] 2024-11-20T09:00:30,948 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T09:00:30,948 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T09:00:30,949 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52798, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T09:00:30,949 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ed8cfb9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T09:00:30,950 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6594bb9c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T09:00:30,950 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request be1eb2620e0e,44373,-1 for getting cluster id 2024-11-20T09:00:30,950 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T09:00:30,950 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'caf50035-b19e-4cfd-90ea-dca559866882' 2024-11-20T09:00:30,950 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T09:00:30,951 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "caf50035-b19e-4cfd-90ea-dca559866882" 2024-11-20T09:00:30,951 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79853070, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T09:00:30,951 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [be1eb2620e0e,44373,-1] 2024-11-20T09:00:30,951 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T09:00:30,951 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T09:00:30,951 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52808, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T09:00:30,952 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3fa64b96, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T09:00:30,952 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61deb417, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T09:00:30,952 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request be1eb2620e0e,44373,-1 for getting cluster id 2024-11-20T09:00:30,953 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T09:00:30,953 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'caf50035-b19e-4cfd-90ea-dca559866882' 2024-11-20T09:00:30,953 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T09:00:30,953 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "caf50035-b19e-4cfd-90ea-dca559866882" 2024-11-20T09:00:30,953 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@39de3a1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T09:00:30,953 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [be1eb2620e0e,44373,-1] 2024-11-20T09:00:30,953 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T09:00:30,954 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T09:00:30,954 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52830, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T09:00:30,954 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@214f4a07, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T09:00:30,955 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47f17e9d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T09:00:30,955 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request be1eb2620e0e,44373,-1 for getting cluster id 2024-11-20T09:00:30,955 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T09:00:30,956 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'caf50035-b19e-4cfd-90ea-dca559866882' 2024-11-20T09:00:30,956 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T09:00:30,956 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "caf50035-b19e-4cfd-90ea-dca559866882" 2024-11-20T09:00:30,956 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42845afe, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T09:00:30,956 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [be1eb2620e0e,44373,-1] 2024-11-20T09:00:30,957 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T09:00:30,957 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T09:00:30,957 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52852, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T09:00:30,958 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7926d9c6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T09:00:30,958 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@675e61a9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T09:00:30,958 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request be1eb2620e0e,44373,-1 for getting cluster id 2024-11-20T09:00:30,958 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T09:00:30,959 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'caf50035-b19e-4cfd-90ea-dca559866882' 2024-11-20T09:00:30,959 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T09:00:30,959 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "caf50035-b19e-4cfd-90ea-dca559866882" 2024-11-20T09:00:30,959 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@540e688e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T09:00:30,959 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [be1eb2620e0e,44373,-1] 2024-11-20T09:00:30,959 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T09:00:30,959 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T09:00:30,960 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52870, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T09:00:30,960 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ce444d5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T09:00:30,961 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46eaeeea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T09:00:30,961 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request be1eb2620e0e,44373,-1 for getting cluster id 2024-11-20T09:00:30,961 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T09:00:30,961 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'caf50035-b19e-4cfd-90ea-dca559866882' 2024-11-20T09:00:30,961 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T09:00:30,962 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "caf50035-b19e-4cfd-90ea-dca559866882" 2024-11-20T09:00:30,962 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4730c0d2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T09:00:30,962 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [be1eb2620e0e,44373,-1] 2024-11-20T09:00:30,962 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T09:00:30,962 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T09:00:30,962 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52892, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T09:00:30,963 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6d8fea03, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T09:00:30,963 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6bee866e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T09:00:30,963 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request be1eb2620e0e,44373,-1 for getting cluster id 2024-11-20T09:00:30,963 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T09:00:30,964 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'caf50035-b19e-4cfd-90ea-dca559866882' 2024-11-20T09:00:30,964 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T09:00:30,964 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "caf50035-b19e-4cfd-90ea-dca559866882" 2024-11-20T09:00:30,964 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@31a75de0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T09:00:30,964 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [be1eb2620e0e,44373,-1] 2024-11-20T09:00:30,964 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T09:00:30,964 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T09:00:30,965 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52904, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T09:00:30,965 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e3ac115, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T09:00:30,966 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@148ac527, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T09:00:30,966 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request be1eb2620e0e,44373,-1 for getting cluster id 2024-11-20T09:00:30,966 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T09:00:30,967 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'caf50035-b19e-4cfd-90ea-dca559866882' 2024-11-20T09:00:30,967 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T09:00:30,967 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "caf50035-b19e-4cfd-90ea-dca559866882" 2024-11-20T09:00:30,967 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6bd5627f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T09:00:30,967 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [be1eb2620e0e,44373,-1] 2024-11-20T09:00:30,967 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T09:00:30,967 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T09:00:30,968 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52928, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T09:00:30,968 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1687c764, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T09:00:30,968 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4199deaf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T09:00:30,969 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request be1eb2620e0e,44373,-1 for getting cluster id 2024-11-20T09:00:30,969 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T09:00:30,969 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'caf50035-b19e-4cfd-90ea-dca559866882' 2024-11-20T09:00:30,969 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T09:00:30,969 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "caf50035-b19e-4cfd-90ea-dca559866882" 2024-11-20T09:00:30,970 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2c516c3a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T09:00:30,970 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [be1eb2620e0e,44373,-1] 2024-11-20T09:00:30,970 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T09:00:30,970 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T09:00:30,970 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52952, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T09:00:30,971 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6633d710, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T09:00:30,971 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1e1d0fb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T09:00:30,971 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request be1eb2620e0e,44373,-1 for getting cluster id 2024-11-20T09:00:30,971 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T09:00:30,972 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'caf50035-b19e-4cfd-90ea-dca559866882' 2024-11-20T09:00:30,972 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T09:00:30,972 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "caf50035-b19e-4cfd-90ea-dca559866882" 2024-11-20T09:00:30,972 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@511df334, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T09:00:30,972 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [be1eb2620e0e,44373,-1] 2024-11-20T09:00:30,972 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T09:00:30,972 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T09:00:30,973 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52970, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T09:00:30,973 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@17a3f714, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T09:00:30,974 DEBUG [Thread-1289 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T09:00:30,975 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=be1eb2620e0e,34223,1732093159960, seqNum=-1] 2024-11-20T09:00:30,975 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T09:00:30,976 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58448, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T09:00:30,977 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2] 2024-11-20T09:00:30,979 DEBUG [Thread-1297 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T09:00:30,979 DEBUG [Thread-1291 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T09:00:30,979 DEBUG [Thread-1306 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T09:00:30,979 DEBUG [Thread-1295 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T09:00:30,980 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=be1eb2620e0e,34223,1732093159960, seqNum=-1] 2024-11-20T09:00:30,980 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=be1eb2620e0e,34223,1732093159960, seqNum=-1] 2024-11-20T09:00:30,980 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=be1eb2620e0e,34223,1732093159960, seqNum=-1] 2024-11-20T09:00:30,980 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=be1eb2620e0e,34223,1732093159960, seqNum=-1] 2024-11-20T09:00:30,980 DEBUG [Thread-1300 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T09:00:30,980 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T09:00:30,980 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T09:00:30,980 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T09:00:30,980 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T09:00:30,980 DEBUG [Thread-1293 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T09:00:30,981 DEBUG [Thread-1308 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T09:00:30,981 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58454, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T09:00:30,981 DEBUG [Thread-1300 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=be1eb2620e0e,34223,1732093159960, seqNum=-1] 2024-11-20T09:00:30,981 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=be1eb2620e0e,34223,1732093159960, seqNum=-1] 2024-11-20T09:00:30,981 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58468, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T09:00:30,981 DEBUG [Thread-1302 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T09:00:30,982 DEBUG [Thread-1308 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=be1eb2620e0e,34223,1732093159960, seqNum=-1] 2024-11-20T09:00:30,982 DEBUG [Thread-1300 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T09:00:30,982 DEBUG [Thread-1308 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T09:00:30,982 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T09:00:30,982 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58452, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T09:00:30,982 DEBUG [Thread-1304 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T09:00:30,982 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58486, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T09:00:30,983 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58484, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T09:00:30,983 DEBUG [Thread-1302 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=be1eb2620e0e,34223,1732093159960, seqNum=-1] 2024-11-20T09:00:30,983 DEBUG [Thread-1302 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T09:00:30,983 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58510, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T09:00:30,983 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58506, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T09:00:30,983 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=be1eb2620e0e,34223,1732093159960, seqNum=-1] 2024-11-20T09:00:30,983 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T09:00:30,984 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58526, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T09:00:30,984 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2] 2024-11-20T09:00:30,985 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2] 2024-11-20T09:00:30,985 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2] 2024-11-20T09:00:30,986 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2] 2024-11-20T09:00:30,987 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2] 2024-11-20T09:00:30,987 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2] 2024-11-20T09:00:30,988 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2] 2024-11-20T09:00:30,988 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58528, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T09:00:30,988 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2] 2024-11-20T09:00:30,989 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2] 2024-11-20T09:00:30,992 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T09:00:30,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] procedure2.ProcedureExecutor(1139): Stored pid=60, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees 2024-11-20T09:00:30,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-20T09:00:30,994 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=60, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T09:00:30,995 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=60, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T09:00:30,995 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T09:00:31,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(8855): Flush requested on 11f6fecb00005c6636e7ea541ff70dfb 2024-11-20T09:00:31,018 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T09:00:31,018 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 11f6fecb00005c6636e7ea541ff70dfb 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-20T09:00:31,050 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/A/1b201078cdb94e56b1f2af61c2f11a4e is 50, key is test_row_0/A:col10/1732093231018/Put/seqid=0 2024-11-20T09:00:31,054 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742107_1283 (size=12001) 2024-11-20T09:00:31,055 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/A/1b201078cdb94e56b1f2af61c2f11a4e 2024-11-20T09:00:31,078 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/B/c5c0794e81dc4f188b8762191241dae0 is 50, key is test_row_0/B:col10/1732093231018/Put/seqid=0 2024-11-20T09:00:31,082 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742108_1284 (size=12001) 2024-11-20T09:00:31,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-20T09:00:31,103 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:31,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58454 deadline: 1732093291098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:31,103 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:31,103 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:31,103 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:00:31,104 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:31,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58448 deadline: 1732093291098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:31,104 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:31,104 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:31,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58510 deadline: 1732093291099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:31,104 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:31,104 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:00:31,105 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:31,105 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:31,105 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:00:31,105 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:31,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58452 deadline: 1732093291100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:31,105 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:31,105 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:31,105 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:31,105 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:00:31,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58468 deadline: 1732093291101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:31,106 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:31,106 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:31,106 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:00:31,148 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-20T09:00:31,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb. 2024-11-20T09:00:31,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb. as already flushing 2024-11-20T09:00:31,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb. 2024-11-20T09:00:31,149 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 11f6fecb00005c6636e7ea541ff70dfb, NAME => 'TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:31,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 11f6fecb00005c6636e7ea541ff70dfb, NAME => 'TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:31,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 11f6fecb00005c6636e7ea541ff70dfb, NAME => 'TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 11f6fecb00005c6636e7ea541ff70dfb, NAME => 'TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:31,209 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:31,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58454 deadline: 1732093291206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:31,210 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:31,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58448 deadline: 1732093291207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:31,210 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:31,210 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:31,210 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:00:31,210 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:31,210 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:31,210 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:31,210 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:00:31,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58510 deadline: 1732093291207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:31,210 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:31,210 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:31,210 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:00:31,218 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:31,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58452 deadline: 1732093291217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:31,219 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:31,219 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:31,219 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:31,219 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:00:31,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58468 deadline: 1732093291217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:31,219 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:31,219 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:31,219 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:00:31,302 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-20T09:00:31,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb. 2024-11-20T09:00:31,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb. as already flushing 2024-11-20T09:00:31,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb. 2024-11-20T09:00:31,303 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 11f6fecb00005c6636e7ea541ff70dfb, NAME => 'TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:31,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 11f6fecb00005c6636e7ea541ff70dfb, NAME => 'TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:31,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 11f6fecb00005c6636e7ea541ff70dfb, NAME => 'TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 11f6fecb00005c6636e7ea541ff70dfb, NAME => 'TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:31,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-20T09:00:31,420 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:31,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58454 deadline: 1732093291417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:31,421 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:31,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58510 deadline: 1732093291417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:31,421 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:31,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58448 deadline: 1732093291417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:31,421 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:31,421 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:31,422 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:00:31,422 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:31,422 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:31,422 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:00:31,422 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:31,422 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:31,422 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:00:31,427 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:31,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58468 deadline: 1732093291427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:31,427 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:31,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58452 deadline: 1732093291427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:31,428 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:31,428 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:31,428 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:00:31,429 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:31,429 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:31,429 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:00:31,457 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-20T09:00:31,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb. 2024-11-20T09:00:31,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb. as already flushing 2024-11-20T09:00:31,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb. 2024-11-20T09:00:31,457 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 11f6fecb00005c6636e7ea541ff70dfb, NAME => 'TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:31,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 11f6fecb00005c6636e7ea541ff70dfb, NAME => 'TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:31,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 11f6fecb00005c6636e7ea541ff70dfb, NAME => 'TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 11f6fecb00005c6636e7ea541ff70dfb, NAME => 'TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:31,483 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/B/c5c0794e81dc4f188b8762191241dae0 2024-11-20T09:00:31,512 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/C/93564b184a244cfd964dc54e1e3b1625 is 50, key is test_row_0/C:col10/1732093231018/Put/seqid=0 2024-11-20T09:00:31,542 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742109_1285 (size=12001) 2024-11-20T09:00:31,611 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-20T09:00:31,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb. 2024-11-20T09:00:31,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb. as already flushing 2024-11-20T09:00:31,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb. 2024-11-20T09:00:31,611 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 11f6fecb00005c6636e7ea541ff70dfb, NAME => 'TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:31,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 11f6fecb00005c6636e7ea541ff70dfb, NAME => 'TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:31,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 11f6fecb00005c6636e7ea541ff70dfb, NAME => 'TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 11f6fecb00005c6636e7ea541ff70dfb, NAME => 'TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:31,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-20T09:00:31,729 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:31,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58448 deadline: 1732093291727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:31,729 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:31,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58510 deadline: 1732093291727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:31,729 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:31,729 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:31,729 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:00:31,729 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:31,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58454 deadline: 1732093291727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:31,730 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:31,730 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:31,730 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:00:31,730 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:31,730 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:31,730 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:00:31,738 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:31,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58468 deadline: 1732093291736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:31,739 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:31,739 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:31,739 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:00:31,739 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:31,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58452 deadline: 1732093291737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:31,740 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:31,740 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:31,740 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:00:31,764 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-20T09:00:31,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb. 2024-11-20T09:00:31,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb. as already flushing 2024-11-20T09:00:31,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb. 2024-11-20T09:00:31,765 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 11f6fecb00005c6636e7ea541ff70dfb, NAME => 'TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:31,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 11f6fecb00005c6636e7ea541ff70dfb, NAME => 'TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:31,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 11f6fecb00005c6636e7ea541ff70dfb, NAME => 'TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 11f6fecb00005c6636e7ea541ff70dfb, NAME => 'TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:31,917 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-20T09:00:31,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb. 2024-11-20T09:00:31,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb. as already flushing 2024-11-20T09:00:31,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb. 2024-11-20T09:00:31,918 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 11f6fecb00005c6636e7ea541ff70dfb, NAME => 'TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:31,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 11f6fecb00005c6636e7ea541ff70dfb, NAME => 'TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:31,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 11f6fecb00005c6636e7ea541ff70dfb, NAME => 'TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 11f6fecb00005c6636e7ea541ff70dfb, NAME => 'TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:31,943 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/C/93564b184a244cfd964dc54e1e3b1625 2024-11-20T09:00:31,948 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/A/1b201078cdb94e56b1f2af61c2f11a4e as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/A/1b201078cdb94e56b1f2af61c2f11a4e 2024-11-20T09:00:31,953 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/A/1b201078cdb94e56b1f2af61c2f11a4e, entries=150, sequenceid=14, filesize=11.7 K 2024-11-20T09:00:31,954 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/B/c5c0794e81dc4f188b8762191241dae0 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/B/c5c0794e81dc4f188b8762191241dae0 2024-11-20T09:00:31,958 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/B/c5c0794e81dc4f188b8762191241dae0, entries=150, sequenceid=14, filesize=11.7 K 2024-11-20T09:00:31,959 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/C/93564b184a244cfd964dc54e1e3b1625 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/C/93564b184a244cfd964dc54e1e3b1625 2024-11-20T09:00:31,965 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/C/93564b184a244cfd964dc54e1e3b1625, entries=150, sequenceid=14, filesize=11.7 K 2024-11-20T09:00:31,966 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 11f6fecb00005c6636e7ea541ff70dfb in 948ms, sequenceid=14, compaction requested=false 2024-11-20T09:00:31,966 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-20T09:00:31,967 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 11f6fecb00005c6636e7ea541ff70dfb: 2024-11-20T09:00:32,071 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-20T09:00:32,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb. 2024-11-20T09:00:32,071 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2902): Flushing 11f6fecb00005c6636e7ea541ff70dfb 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-20T09:00:32,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/A/ebc60d666f4a4ea7ba605455f377dcdc is 50, key is test_row_0/A:col10/1732093231098/Put/seqid=0 2024-11-20T09:00:32,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-20T09:00:32,158 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742110_1286 (size=12001) 2024-11-20T09:00:32,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(8855): Flush requested on 11f6fecb00005c6636e7ea541ff70dfb 2024-11-20T09:00:32,239 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb. as already flushing 2024-11-20T09:00:32,252 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:32,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58468 deadline: 1732093292247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:32,253 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:32,253 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:32,253 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:32,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58448 deadline: 1732093292249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:32,253 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:00:32,253 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:32,253 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:32,253 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:00:32,255 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:32,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58510 deadline: 1732093292250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:32,255 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:32,255 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:32,255 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:00:32,257 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:32,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58454 deadline: 1732093292251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:32,258 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:32,258 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:32,258 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:32,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58452 deadline: 1732093292252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:32,258 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:00:32,258 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:32,258 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:32,258 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:00:32,359 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:32,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58448 deadline: 1732093292357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:32,360 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:32,360 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:32,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58510 deadline: 1732093292357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:32,360 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:32,360 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:00:32,360 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:32,360 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:32,361 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:00:32,371 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:32,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58454 deadline: 1732093292368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:32,372 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:32,372 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:32,372 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:00:32,372 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:32,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58452 deadline: 1732093292368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:32,372 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:32,372 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:32,372 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:00:32,559 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/A/ebc60d666f4a4ea7ba605455f377dcdc 2024-11-20T09:00:32,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/B/f4b3551841694e33a70f3f73315f4584 is 50, key is test_row_0/B:col10/1732093231098/Put/seqid=0 2024-11-20T09:00:32,567 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:32,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58448 deadline: 1732093292567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:32,569 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:32,569 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:32,569 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:00:32,571 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:32,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58510 deadline: 1732093292568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:32,572 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:32,572 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:32,572 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:00:32,579 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:32,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58452 deadline: 1732093292577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:32,580 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:32,580 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:32,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58454 deadline: 1732093292577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:32,580 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:32,580 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:00:32,580 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:32,580 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:32,580 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:00:32,593 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742111_1287 (size=12001) 2024-11-20T09:00:32,594 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/B/f4b3551841694e33a70f3f73315f4584 2024-11-20T09:00:32,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/C/9de830fc02404ecf9b1048b24acdf5b5 is 50, key is test_row_0/C:col10/1732093231098/Put/seqid=0 2024-11-20T09:00:32,606 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742112_1288 (size=12001) 2024-11-20T09:00:32,879 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:32,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58448 deadline: 1732093292877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:32,880 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:32,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58510 deadline: 1732093292877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:32,880 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:32,880 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:32,880 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:00:32,880 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:32,880 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:32,880 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:00:32,890 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:32,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58452 deadline: 1732093292886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:32,890 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:32,890 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:32,890 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:32,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58454 deadline: 1732093292886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:32,890 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:00:32,890 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:32,890 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:32,891 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:00:33,007 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/C/9de830fc02404ecf9b1048b24acdf5b5 2024-11-20T09:00:33,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/A/ebc60d666f4a4ea7ba605455f377dcdc as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/A/ebc60d666f4a4ea7ba605455f377dcdc 2024-11-20T09:00:33,016 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/A/ebc60d666f4a4ea7ba605455f377dcdc, entries=150, sequenceid=37, filesize=11.7 K 2024-11-20T09:00:33,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/B/f4b3551841694e33a70f3f73315f4584 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/B/f4b3551841694e33a70f3f73315f4584 2024-11-20T09:00:33,021 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/B/f4b3551841694e33a70f3f73315f4584, entries=150, sequenceid=37, filesize=11.7 K 2024-11-20T09:00:33,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/C/9de830fc02404ecf9b1048b24acdf5b5 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/C/9de830fc02404ecf9b1048b24acdf5b5 2024-11-20T09:00:33,025 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/C/9de830fc02404ecf9b1048b24acdf5b5, entries=150, sequenceid=37, filesize=11.7 K 2024-11-20T09:00:33,026 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 11f6fecb00005c6636e7ea541ff70dfb in 955ms, sequenceid=37, compaction requested=false 2024-11-20T09:00:33,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2603): Flush status journal for 11f6fecb00005c6636e7ea541ff70dfb: 2024-11-20T09:00:33,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb. 2024-11-20T09:00:33,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=61 2024-11-20T09:00:33,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.HMaster(4169): Remote procedure done, pid=61 2024-11-20T09:00:33,030 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-11-20T09:00:33,030 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0330 sec 2024-11-20T09:00:33,032 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees in 2.0390 sec 2024-11-20T09:00:33,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-20T09:00:33,137 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T09:00:33,137 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T09:00:33,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] procedure2.ProcedureExecutor(1139): Stored pid=62, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees 2024-11-20T09:00:33,139 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=62, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T09:00:33,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-20T09:00:33,140 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=62, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T09:00:33,140 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T09:00:33,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-20T09:00:33,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(8855): Flush requested on 11f6fecb00005c6636e7ea541ff70dfb 2024-11-20T09:00:33,260 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T09:00:33,260 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 11f6fecb00005c6636e7ea541ff70dfb 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-20T09:00:33,265 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/A/e909f16849d049d08d098ae69809ddeb is 50, key is test_row_0/A:col10/1732093232244/Put/seqid=0 2024-11-20T09:00:33,269 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742113_1289 (size=14341) 2024-11-20T09:00:33,270 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/A/e909f16849d049d08d098ae69809ddeb 2024-11-20T09:00:33,277 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/B/7b28b22d5df941858c7448a22c88959a is 50, key is test_row_0/B:col10/1732093232244/Put/seqid=0 2024-11-20T09:00:33,284 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742114_1290 (size=12001) 2024-11-20T09:00:33,284 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/B/7b28b22d5df941858c7448a22c88959a 2024-11-20T09:00:33,292 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/C/628fb7b69bdb44d894710bf71ed5a1c4 is 50, key is test_row_0/C:col10/1732093232244/Put/seqid=0 2024-11-20T09:00:33,293 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-20T09:00:33,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb. 2024-11-20T09:00:33,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb. as already flushing 2024-11-20T09:00:33,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb. 2024-11-20T09:00:33,293 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 11f6fecb00005c6636e7ea541ff70dfb, NAME => 'TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:33,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 11f6fecb00005c6636e7ea541ff70dfb, NAME => 'TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:33,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 11f6fecb00005c6636e7ea541ff70dfb, NAME => 'TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 11f6fecb00005c6636e7ea541ff70dfb, NAME => 'TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:33,296 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742115_1291 (size=12001) 2024-11-20T09:00:33,297 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/C/628fb7b69bdb44d894710bf71ed5a1c4 2024-11-20T09:00:33,301 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/A/e909f16849d049d08d098ae69809ddeb as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/A/e909f16849d049d08d098ae69809ddeb 2024-11-20T09:00:33,305 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/A/e909f16849d049d08d098ae69809ddeb, entries=200, sequenceid=51, filesize=14.0 K 2024-11-20T09:00:33,306 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/B/7b28b22d5df941858c7448a22c88959a as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/B/7b28b22d5df941858c7448a22c88959a 2024-11-20T09:00:33,310 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/B/7b28b22d5df941858c7448a22c88959a, entries=150, sequenceid=51, filesize=11.7 K 2024-11-20T09:00:33,310 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/C/628fb7b69bdb44d894710bf71ed5a1c4 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/C/628fb7b69bdb44d894710bf71ed5a1c4 2024-11-20T09:00:33,315 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/C/628fb7b69bdb44d894710bf71ed5a1c4, entries=150, sequenceid=51, filesize=11.7 K 2024-11-20T09:00:33,315 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=73.80 KB/75570 for 11f6fecb00005c6636e7ea541ff70dfb in 55ms, sequenceid=51, compaction requested=true 2024-11-20T09:00:33,315 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 11f6fecb00005c6636e7ea541ff70dfb: 2024-11-20T09:00:33,315 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 11f6fecb00005c6636e7ea541ff70dfb:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T09:00:33,316 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T09:00:33,316 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 11f6fecb00005c6636e7ea541ff70dfb:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T09:00:33,316 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T09:00:33,316 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 11f6fecb00005c6636e7ea541ff70dfb:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T09:00:33,316 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T09:00:33,316 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T09:00:33,316 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T09:00:33,316 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38343 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T09:00:33,316 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T09:00:33,317 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HStore(1541): 11f6fecb00005c6636e7ea541ff70dfb/A is initiating minor compaction (all files) 2024-11-20T09:00:33,317 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1541): 11f6fecb00005c6636e7ea541ff70dfb/B is initiating minor compaction (all files) 2024-11-20T09:00:33,317 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 11f6fecb00005c6636e7ea541ff70dfb/A in TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb. 2024-11-20T09:00:33,317 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 11f6fecb00005c6636e7ea541ff70dfb/B in TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb. 2024-11-20T09:00:33,317 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/A/1b201078cdb94e56b1f2af61c2f11a4e, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/A/ebc60d666f4a4ea7ba605455f377dcdc, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/A/e909f16849d049d08d098ae69809ddeb] into tmpdir=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp, totalSize=37.4 K 2024-11-20T09:00:33,317 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/B/c5c0794e81dc4f188b8762191241dae0, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/B/f4b3551841694e33a70f3f73315f4584, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/B/7b28b22d5df941858c7448a22c88959a] into tmpdir=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp, totalSize=35.2 K 2024-11-20T09:00:33,318 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1b201078cdb94e56b1f2af61c2f11a4e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1732093231016 2024-11-20T09:00:33,318 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting c5c0794e81dc4f188b8762191241dae0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1732093231016 2024-11-20T09:00:33,318 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.Compactor(225): Compacting ebc60d666f4a4ea7ba605455f377dcdc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1732093231083 2024-11-20T09:00:33,319 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting f4b3551841694e33a70f3f73315f4584, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1732093231083 2024-11-20T09:00:33,319 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.Compactor(225): Compacting e909f16849d049d08d098ae69809ddeb, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1732093232244 2024-11-20T09:00:33,319 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting 7b28b22d5df941858c7448a22c88959a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1732093232244 2024-11-20T09:00:33,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(8855): Flush requested on 11f6fecb00005c6636e7ea541ff70dfb 2024-11-20T09:00:33,319 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T09:00:33,319 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 11f6fecb00005c6636e7ea541ff70dfb 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-20T09:00:33,339 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 11f6fecb00005c6636e7ea541ff70dfb#A#compaction#241 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-20T09:00:33,339 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 11f6fecb00005c6636e7ea541ff70dfb#B#compaction#240 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T09:00:33,339 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/A/9ef3f27fdf994e5988897c44285ba1ec is 50, key is test_row_0/A:col10/1732093232244/Put/seqid=0 2024-11-20T09:00:33,340 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/B/2e6863d840a64ee4aacf05869b593f42 is 50, key is test_row_0/B:col10/1732093232244/Put/seqid=0 2024-11-20T09:00:33,346 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/A/30f9fa05ff3945e29019e13f780b9b49 is 50, key is test_row_0/A:col10/1732093233317/Put/seqid=0 2024-11-20T09:00:33,349 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742117_1293 (size=12104) 2024-11-20T09:00:33,357 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742116_1292 (size=12104) 2024-11-20T09:00:33,365 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742118_1294 (size=12001) 2024-11-20T09:00:33,399 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:33,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58448 deadline: 1732093293392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:33,399 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:33,399 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:33,399 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:33,399 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:00:33,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58468 deadline: 1732093293394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:33,399 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:33,400 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:33,400 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:33,400 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:00:33,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58452 deadline: 1732093293397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:33,400 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:33,400 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:33,400 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:00:33,400 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:33,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58454 deadline: 1732093293397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:33,400 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:33,400 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:33,400 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:00:33,403 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:33,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58510 deadline: 1732093293399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:33,404 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:33,404 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:33,404 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:00:33,446 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-20T09:00:33,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb. 2024-11-20T09:00:33,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb. as already flushing 2024-11-20T09:00:33,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb. 2024-11-20T09:00:33,447 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 11f6fecb00005c6636e7ea541ff70dfb, NAME => 'TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:33,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 11f6fecb00005c6636e7ea541ff70dfb, NAME => 'TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:33,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 11f6fecb00005c6636e7ea541ff70dfb, NAME => 'TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 11f6fecb00005c6636e7ea541ff70dfb, NAME => 'TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:33,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-20T09:00:33,510 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:33,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58448 deadline: 1732093293507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:33,511 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:33,511 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:33,511 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:33,511 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:00:33,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58468 deadline: 1732093293507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:33,511 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:33,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58510 deadline: 1732093293507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:33,511 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:33,511 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:33,511 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:00:33,511 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:33,511 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:33,511 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:00:33,599 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-20T09:00:33,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb. 2024-11-20T09:00:33,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb. as already flushing 2024-11-20T09:00:33,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb. 2024-11-20T09:00:33,600 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 11f6fecb00005c6636e7ea541ff70dfb, NAME => 'TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:33,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 11f6fecb00005c6636e7ea541ff70dfb, NAME => 'TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:33,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 11f6fecb00005c6636e7ea541ff70dfb, NAME => 'TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 11f6fecb00005c6636e7ea541ff70dfb, NAME => 'TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:33,719 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:33,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58448 deadline: 1732093293717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:33,719 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:33,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58468 deadline: 1732093293717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:33,719 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:33,720 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:33,720 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:00:33,720 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:33,720 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:33,720 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:33,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58510 deadline: 1732093293717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:33,720 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:00:33,720 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:33,720 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:33,720 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:00:33,753 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-20T09:00:33,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb. 2024-11-20T09:00:33,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb. as already flushing 2024-11-20T09:00:33,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb. 2024-11-20T09:00:33,753 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 11f6fecb00005c6636e7ea541ff70dfb, NAME => 'TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:33,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 11f6fecb00005c6636e7ea541ff70dfb, NAME => 'TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:33,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 11f6fecb00005c6636e7ea541ff70dfb, NAME => 'TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 11f6fecb00005c6636e7ea541ff70dfb, NAME => 'TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:33,756 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/B/2e6863d840a64ee4aacf05869b593f42 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/B/2e6863d840a64ee4aacf05869b593f42 2024-11-20T09:00:33,766 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/A/30f9fa05ff3945e29019e13f780b9b49 2024-11-20T09:00:33,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-20T09:00:33,768 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/A/9ef3f27fdf994e5988897c44285ba1ec as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/A/9ef3f27fdf994e5988897c44285ba1ec 2024-11-20T09:00:33,771 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 11f6fecb00005c6636e7ea541ff70dfb/B of 11f6fecb00005c6636e7ea541ff70dfb into 2e6863d840a64ee4aacf05869b593f42(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T09:00:33,771 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 11f6fecb00005c6636e7ea541ff70dfb: 2024-11-20T09:00:33,771 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., storeName=11f6fecb00005c6636e7ea541ff70dfb/B, priority=13, startTime=1732093233316; duration=0sec 2024-11-20T09:00:33,771 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T09:00:33,771 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 11f6fecb00005c6636e7ea541ff70dfb:B 2024-11-20T09:00:33,771 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T09:00:33,774 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T09:00:33,774 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1541): 11f6fecb00005c6636e7ea541ff70dfb/C is initiating minor compaction (all files) 2024-11-20T09:00:33,774 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 11f6fecb00005c6636e7ea541ff70dfb/C in TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb. 2024-11-20T09:00:33,774 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/C/93564b184a244cfd964dc54e1e3b1625, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/C/9de830fc02404ecf9b1048b24acdf5b5, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/C/628fb7b69bdb44d894710bf71ed5a1c4] into tmpdir=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp, totalSize=35.2 K 2024-11-20T09:00:33,775 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting 93564b184a244cfd964dc54e1e3b1625, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1732093231016 2024-11-20T09:00:33,777 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting 9de830fc02404ecf9b1048b24acdf5b5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1732093231083 2024-11-20T09:00:33,777 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting 628fb7b69bdb44d894710bf71ed5a1c4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1732093232244 2024-11-20T09:00:33,777 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 11f6fecb00005c6636e7ea541ff70dfb/A of 11f6fecb00005c6636e7ea541ff70dfb into 9ef3f27fdf994e5988897c44285ba1ec(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T09:00:33,777 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 11f6fecb00005c6636e7ea541ff70dfb: 2024-11-20T09:00:33,777 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., storeName=11f6fecb00005c6636e7ea541ff70dfb/A, priority=13, startTime=1732093233315; duration=0sec 2024-11-20T09:00:33,777 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T09:00:33,777 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 11f6fecb00005c6636e7ea541ff70dfb:A 2024-11-20T09:00:33,788 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/B/3de3f7d0a4214a028ed7fd0dd3494909 is 50, key is test_row_0/B:col10/1732093233317/Put/seqid=0 2024-11-20T09:00:33,788 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 11f6fecb00005c6636e7ea541ff70dfb#C#compaction#244 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T09:00:33,789 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/C/b463a22430364744b162d403b2d380a7 is 50, key is test_row_0/C:col10/1732093232244/Put/seqid=0 2024-11-20T09:00:33,798 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742119_1295 (size=12001) 2024-11-20T09:00:33,801 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/B/3de3f7d0a4214a028ed7fd0dd3494909 2024-11-20T09:00:33,809 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/C/b423d82390304e08a6b1ca9fc8eecdb0 is 50, key is test_row_0/C:col10/1732093233317/Put/seqid=0 2024-11-20T09:00:33,812 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742120_1296 (size=12104) 2024-11-20T09:00:33,818 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/C/b463a22430364744b162d403b2d380a7 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/C/b463a22430364744b162d403b2d380a7 2024-11-20T09:00:33,824 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 11f6fecb00005c6636e7ea541ff70dfb/C of 11f6fecb00005c6636e7ea541ff70dfb into b463a22430364744b162d403b2d380a7(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T09:00:33,825 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 11f6fecb00005c6636e7ea541ff70dfb: 2024-11-20T09:00:33,825 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., storeName=11f6fecb00005c6636e7ea541ff70dfb/C, priority=13, startTime=1732093233316; duration=0sec 2024-11-20T09:00:33,825 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T09:00:33,825 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 11f6fecb00005c6636e7ea541ff70dfb:C 2024-11-20T09:00:33,825 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742121_1297 (size=12001) 2024-11-20T09:00:33,826 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/C/b423d82390304e08a6b1ca9fc8eecdb0 2024-11-20T09:00:33,832 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/A/30f9fa05ff3945e29019e13f780b9b49 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/A/30f9fa05ff3945e29019e13f780b9b49 2024-11-20T09:00:33,835 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/A/30f9fa05ff3945e29019e13f780b9b49, entries=150, sequenceid=66, filesize=11.7 K 2024-11-20T09:00:33,836 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/B/3de3f7d0a4214a028ed7fd0dd3494909 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/B/3de3f7d0a4214a028ed7fd0dd3494909 2024-11-20T09:00:33,839 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/B/3de3f7d0a4214a028ed7fd0dd3494909, entries=150, sequenceid=66, filesize=11.7 K 2024-11-20T09:00:33,839 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/C/b423d82390304e08a6b1ca9fc8eecdb0 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/C/b423d82390304e08a6b1ca9fc8eecdb0 2024-11-20T09:00:33,843 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/C/b423d82390304e08a6b1ca9fc8eecdb0, entries=150, sequenceid=66, filesize=11.7 K 2024-11-20T09:00:33,844 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for 11f6fecb00005c6636e7ea541ff70dfb in 525ms, sequenceid=66, compaction requested=false 2024-11-20T09:00:33,844 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 11f6fecb00005c6636e7ea541ff70dfb: 2024-11-20T09:00:33,907 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-20T09:00:33,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb. 2024-11-20T09:00:33,907 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2902): Flushing 11f6fecb00005c6636e7ea541ff70dfb 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-20T09:00:33,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/A/15806525791e4e58818dd7d6c9dfa5df is 50, key is test_row_0/A:col10/1732093233397/Put/seqid=0 2024-11-20T09:00:33,919 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742122_1298 (size=12001) 2024-11-20T09:00:33,951 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-20T09:00:34,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(8855): Flush requested on 11f6fecb00005c6636e7ea541ff70dfb 2024-11-20T09:00:34,029 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb. as already flushing 2024-11-20T09:00:34,073 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:34,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58448 deadline: 1732093294069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:34,073 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:34,074 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:34,074 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:34,074 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:00:34,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58468 deadline: 1732093294069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:34,074 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:34,074 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:34,074 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:00:34,075 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:34,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58510 deadline: 1732093294071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:34,075 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:34,075 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:34,075 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:00:34,179 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:34,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58448 deadline: 1732093294177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:34,179 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:34,180 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:34,180 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:34,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58468 deadline: 1732093294177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:34,180 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:00:34,180 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:34,180 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:34,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58510 deadline: 1732093294177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:34,180 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:34,180 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:00:34,180 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:34,180 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:34,180 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:00:34,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-20T09:00:34,320 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/A/15806525791e4e58818dd7d6c9dfa5df 2024-11-20T09:00:34,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/B/a0bf154b946d49c28b6494d362a65beb is 50, key is test_row_0/B:col10/1732093233397/Put/seqid=0 2024-11-20T09:00:34,335 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742123_1299 (size=12001) 2024-11-20T09:00:34,390 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:34,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58448 deadline: 1732093294387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:34,390 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:34,390 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:34,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58468 deadline: 1732093294387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:34,390 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:34,391 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:00:34,391 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:34,391 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:34,391 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:34,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58510 deadline: 1732093294387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:34,391 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:00:34,391 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:34,391 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:34,391 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:00:34,410 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:34,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58452 deadline: 1732093294407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:34,410 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to be1eb2620e0e,34223,1732093159960 for 'test_row_2' in 11f6fecb00005c6636e7ea541ff70dfb of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2157 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-20T09:00:34,410 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:34,411 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:34,411 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:00:34,420 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:34,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58454 deadline: 1732093294417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:34,420 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to be1eb2620e0e,34223,1732093159960 for 'test_row_0' in 11f6fecb00005c6636e7ea541ff70dfb of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2168 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-20T09:00:34,421 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:34,421 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:34,421 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:00:34,698 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:34,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58448 deadline: 1732093294697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:34,699 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:34,699 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:34,699 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:34,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58468 deadline: 1732093294697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:34,699 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:00:34,699 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:34,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58510 deadline: 1732093294697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:34,699 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:34,699 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:34,699 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:00:34,699 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:34,699 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:34,700 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:00:34,736 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/B/a0bf154b946d49c28b6494d362a65beb 2024-11-20T09:00:34,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/C/b3162f2c44a14e33a260e7c69d9c0c71 is 50, key is test_row_0/C:col10/1732093233397/Put/seqid=0 2024-11-20T09:00:34,756 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742124_1300 (size=12001) 2024-11-20T09:00:35,157 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/C/b3162f2c44a14e33a260e7c69d9c0c71 2024-11-20T09:00:35,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/A/15806525791e4e58818dd7d6c9dfa5df as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/A/15806525791e4e58818dd7d6c9dfa5df 2024-11-20T09:00:35,166 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/A/15806525791e4e58818dd7d6c9dfa5df, entries=150, sequenceid=90, filesize=11.7 K 2024-11-20T09:00:35,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/B/a0bf154b946d49c28b6494d362a65beb as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/B/a0bf154b946d49c28b6494d362a65beb 2024-11-20T09:00:35,172 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/B/a0bf154b946d49c28b6494d362a65beb, entries=150, sequenceid=90, filesize=11.7 K 2024-11-20T09:00:35,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/C/b3162f2c44a14e33a260e7c69d9c0c71 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/C/b3162f2c44a14e33a260e7c69d9c0c71 2024-11-20T09:00:35,177 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/C/b3162f2c44a14e33a260e7c69d9c0c71, entries=150, sequenceid=90, filesize=11.7 K 2024-11-20T09:00:35,178 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for 11f6fecb00005c6636e7ea541ff70dfb in 1271ms, sequenceid=90, compaction requested=true 2024-11-20T09:00:35,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2603): Flush status journal for 11f6fecb00005c6636e7ea541ff70dfb: 2024-11-20T09:00:35,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb. 2024-11-20T09:00:35,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=63 2024-11-20T09:00:35,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.HMaster(4169): Remote procedure done, pid=63 2024-11-20T09:00:35,182 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=62 2024-11-20T09:00:35,182 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0400 sec 2024-11-20T09:00:35,184 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees in 2.0460 sec 2024-11-20T09:00:35,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(8855): Flush requested on 11f6fecb00005c6636e7ea541ff70dfb 2024-11-20T09:00:35,209 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T09:00:35,209 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 11f6fecb00005c6636e7ea541ff70dfb 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-20T09:00:35,216 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/A/c49d7d2f92d94144bfbbcc3f60c6a206 is 50, key is test_row_0/A:col10/1732093235209/Put/seqid=0 2024-11-20T09:00:35,220 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742125_1301 (size=14341) 2024-11-20T09:00:35,240 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:35,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58468 deadline: 1732093295236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:35,241 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:35,241 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:35,241 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:00:35,244 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:35,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58448 deadline: 1732093295238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:35,244 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:35,244 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:35,244 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:00:35,244 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:35,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58510 deadline: 1732093295239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:35,245 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:35,245 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:35,245 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:00:35,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-20T09:00:35,297 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T09:00:35,297 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T09:00:35,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] procedure2.ProcedureExecutor(1139): Stored pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees 2024-11-20T09:00:35,299 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T09:00:35,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-20T09:00:35,300 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T09:00:35,300 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T09:00:35,351 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:35,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58468 deadline: 1732093295347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:35,351 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:35,351 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:35,351 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:35,351 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:00:35,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58448 deadline: 1732093295347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:35,351 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:35,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58510 deadline: 1732093295347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:35,352 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:35,352 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:35,352 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:00:35,352 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:35,352 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:35,352 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:00:35,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-20T09:00:35,452 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-20T09:00:35,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb. 2024-11-20T09:00:35,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb. as already flushing 2024-11-20T09:00:35,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb. 2024-11-20T09:00:35,453 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 11f6fecb00005c6636e7ea541ff70dfb, NAME => 'TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:35,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 11f6fecb00005c6636e7ea541ff70dfb, NAME => 'TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:35,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 11f6fecb00005c6636e7ea541ff70dfb, NAME => 'TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 11f6fecb00005c6636e7ea541ff70dfb, NAME => 'TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:35,560 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:35,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58468 deadline: 1732093295557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:35,560 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:35,561 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:35,561 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:35,561 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:00:35,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58448 deadline: 1732093295557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:35,561 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:35,561 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:35,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58510 deadline: 1732093295557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:35,561 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:35,561 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:00:35,561 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:35,561 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:35,561 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:00:35,605 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-20T09:00:35,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb. 2024-11-20T09:00:35,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb. as already flushing 2024-11-20T09:00:35,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb. 2024-11-20T09:00:35,606 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 11f6fecb00005c6636e7ea541ff70dfb, NAME => 'TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:35,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 11f6fecb00005c6636e7ea541ff70dfb, NAME => 'TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:35,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 11f6fecb00005c6636e7ea541ff70dfb, NAME => 'TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 11f6fecb00005c6636e7ea541ff70dfb, NAME => 'TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:35,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-20T09:00:35,620 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=107 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/A/c49d7d2f92d94144bfbbcc3f60c6a206 2024-11-20T09:00:35,628 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/B/4364ee9b16e64c13a4cd14451a912e16 is 50, key is test_row_0/B:col10/1732093235209/Put/seqid=0 2024-11-20T09:00:35,633 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742126_1302 (size=12001) 2024-11-20T09:00:35,634 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=107 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/B/4364ee9b16e64c13a4cd14451a912e16 2024-11-20T09:00:35,640 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/C/83d7f187dcb54ec78e6ec93f80690f58 is 50, key is test_row_0/C:col10/1732093235209/Put/seqid=0 2024-11-20T09:00:35,644 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742127_1303 (size=12001) 2024-11-20T09:00:35,759 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-20T09:00:35,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb. 2024-11-20T09:00:35,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb. as already flushing 2024-11-20T09:00:35,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb. 2024-11-20T09:00:35,759 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 11f6fecb00005c6636e7ea541ff70dfb, NAME => 'TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:35,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 11f6fecb00005c6636e7ea541ff70dfb, NAME => 'TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:35,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 11f6fecb00005c6636e7ea541ff70dfb, NAME => 'TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 11f6fecb00005c6636e7ea541ff70dfb, NAME => 'TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:35,870 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:35,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58468 deadline: 1732093295867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:35,871 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:35,871 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:35,871 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:35,871 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:00:35,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58448 deadline: 1732093295867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:35,871 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:35,871 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:35,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58510 deadline: 1732093295867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:35,871 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:35,871 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:00:35,871 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:35,871 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:35,871 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:00:35,912 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-20T09:00:35,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb. 2024-11-20T09:00:35,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb. as already flushing 2024-11-20T09:00:35,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb. 2024-11-20T09:00:35,912 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 11f6fecb00005c6636e7ea541ff70dfb, NAME => 'TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:35,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 11f6fecb00005c6636e7ea541ff70dfb, NAME => 'TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:35,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 11f6fecb00005c6636e7ea541ff70dfb, NAME => 'TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 11f6fecb00005c6636e7ea541ff70dfb, NAME => 'TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:35,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-20T09:00:36,045 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=107 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/C/83d7f187dcb54ec78e6ec93f80690f58 2024-11-20T09:00:36,049 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/A/c49d7d2f92d94144bfbbcc3f60c6a206 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/A/c49d7d2f92d94144bfbbcc3f60c6a206 2024-11-20T09:00:36,053 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/A/c49d7d2f92d94144bfbbcc3f60c6a206, entries=200, sequenceid=107, filesize=14.0 K 2024-11-20T09:00:36,054 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/B/4364ee9b16e64c13a4cd14451a912e16 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/B/4364ee9b16e64c13a4cd14451a912e16 2024-11-20T09:00:36,057 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/B/4364ee9b16e64c13a4cd14451a912e16, entries=150, sequenceid=107, filesize=11.7 K 2024-11-20T09:00:36,058 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/C/83d7f187dcb54ec78e6ec93f80690f58 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/C/83d7f187dcb54ec78e6ec93f80690f58 2024-11-20T09:00:36,061 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/C/83d7f187dcb54ec78e6ec93f80690f58, entries=150, sequenceid=107, filesize=11.7 K 2024-11-20T09:00:36,062 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for 11f6fecb00005c6636e7ea541ff70dfb in 853ms, sequenceid=107, compaction requested=true 2024-11-20T09:00:36,062 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 11f6fecb00005c6636e7ea541ff70dfb: 2024-11-20T09:00:36,063 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 11f6fecb00005c6636e7ea541ff70dfb:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T09:00:36,063 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T09:00:36,063 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T09:00:36,063 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 11f6fecb00005c6636e7ea541ff70dfb:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T09:00:36,063 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T09:00:36,063 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 11f6fecb00005c6636e7ea541ff70dfb:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T09:00:36,063 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T09:00:36,063 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T09:00:36,064 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50447 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T09:00:36,064 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HStore(1541): 11f6fecb00005c6636e7ea541ff70dfb/A is initiating minor compaction (all files) 2024-11-20T09:00:36,064 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 11f6fecb00005c6636e7ea541ff70dfb/A in TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb. 2024-11-20T09:00:36,065 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/A/9ef3f27fdf994e5988897c44285ba1ec, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/A/30f9fa05ff3945e29019e13f780b9b49, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/A/15806525791e4e58818dd7d6c9dfa5df, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/A/c49d7d2f92d94144bfbbcc3f60c6a206] into tmpdir=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp, totalSize=49.3 K 2024-11-20T09:00:36,065 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-20T09:00:36,065 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T09:00:36,065 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1541): 11f6fecb00005c6636e7ea541ff70dfb/B is initiating minor compaction (all files) 2024-11-20T09:00:36,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb. 2024-11-20T09:00:36,065 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 11f6fecb00005c6636e7ea541ff70dfb/B in TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb. 2024-11-20T09:00:36,065 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/B/2e6863d840a64ee4aacf05869b593f42, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/B/3de3f7d0a4214a028ed7fd0dd3494909, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/B/a0bf154b946d49c28b6494d362a65beb, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/B/4364ee9b16e64c13a4cd14451a912e16] into tmpdir=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp, totalSize=47.0 K 2024-11-20T09:00:36,065 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2902): Flushing 11f6fecb00005c6636e7ea541ff70dfb 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-20T09:00:36,066 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9ef3f27fdf994e5988897c44285ba1ec, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1732093232244 2024-11-20T09:00:36,066 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting 2e6863d840a64ee4aacf05869b593f42, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1732093232244 2024-11-20T09:00:36,066 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting 3de3f7d0a4214a028ed7fd0dd3494909, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=66, earliestPutTs=1732093233293 2024-11-20T09:00:36,066 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.Compactor(225): Compacting 30f9fa05ff3945e29019e13f780b9b49, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=66, earliestPutTs=1732093233293 2024-11-20T09:00:36,066 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.Compactor(225): Compacting 15806525791e4e58818dd7d6c9dfa5df, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=90, earliestPutTs=1732093233388 2024-11-20T09:00:36,066 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting a0bf154b946d49c28b6494d362a65beb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=90, earliestPutTs=1732093233388 2024-11-20T09:00:36,067 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting 4364ee9b16e64c13a4cd14451a912e16, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=107, earliestPutTs=1732093234066 2024-11-20T09:00:36,067 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.Compactor(225): Compacting c49d7d2f92d94144bfbbcc3f60c6a206, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=107, earliestPutTs=1732093234066 2024-11-20T09:00:36,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/A/aa33f71a11f44f7f925287662e82ba4f is 50, key is test_row_0/A:col10/1732093235228/Put/seqid=0 2024-11-20T09:00:36,082 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742128_1304 (size=12001) 2024-11-20T09:00:36,086 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 11f6fecb00005c6636e7ea541ff70dfb#A#compaction#253 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T09:00:36,087 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/A/8846273420cc4c68bd0ecddfafe4fc5d is 50, key is test_row_0/A:col10/1732093235209/Put/seqid=0 2024-11-20T09:00:36,090 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 11f6fecb00005c6636e7ea541ff70dfb#B#compaction#254 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T09:00:36,091 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/B/a4693a46097b44c2a65f205f76d6f475 is 50, key is test_row_0/B:col10/1732093235209/Put/seqid=0 2024-11-20T09:00:36,103 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742129_1305 (size=12241) 2024-11-20T09:00:36,113 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742130_1306 (size=12241) 2024-11-20T09:00:36,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(8855): Flush requested on 11f6fecb00005c6636e7ea541ff70dfb 2024-11-20T09:00:36,381 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb. as already flushing 2024-11-20T09:00:36,406 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:36,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58510 deadline: 1732093296400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:36,406 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:36,406 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:36,406 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:00:36,408 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:36,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58448 deadline: 1732093296405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:36,409 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:36,409 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:36,409 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:00:36,411 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:36,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58468 deadline: 1732093296406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:36,411 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:36,411 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:36,411 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:00:36,430 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:36,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58452 deadline: 1732093296427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:36,430 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:36,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58454 deadline: 1732093296427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:36,430 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to be1eb2620e0e,34223,1732093159960 for 'test_row_2' in 11f6fecb00005c6636e7ea541ff70dfb of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4178 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-20T09:00:36,431 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:36,431 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:36,431 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to be1eb2620e0e,34223,1732093159960 for 'test_row_0' in 11f6fecb00005c6636e7ea541ff70dfb of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4179 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-20T09:00:36,431 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:00:36,431 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:36,431 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:36,431 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:00:36,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-20T09:00:36,483 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=126 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/A/aa33f71a11f44f7f925287662e82ba4f 2024-11-20T09:00:36,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/B/10c856ff67864213877f268d3471a45d is 50, key is test_row_0/B:col10/1732093235228/Put/seqid=0 2024-11-20T09:00:36,494 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742131_1307 (size=12001) 2024-11-20T09:00:36,508 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/A/8846273420cc4c68bd0ecddfafe4fc5d as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/A/8846273420cc4c68bd0ecddfafe4fc5d 2024-11-20T09:00:36,512 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 11f6fecb00005c6636e7ea541ff70dfb/A of 11f6fecb00005c6636e7ea541ff70dfb into 8846273420cc4c68bd0ecddfafe4fc5d(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T09:00:36,513 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 11f6fecb00005c6636e7ea541ff70dfb: 2024-11-20T09:00:36,513 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., storeName=11f6fecb00005c6636e7ea541ff70dfb/A, priority=12, startTime=1732093236063; duration=0sec 2024-11-20T09:00:36,513 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T09:00:36,513 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 11f6fecb00005c6636e7ea541ff70dfb:A 2024-11-20T09:00:36,513 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T09:00:36,514 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T09:00:36,514 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HStore(1541): 11f6fecb00005c6636e7ea541ff70dfb/C is initiating minor compaction (all files) 2024-11-20T09:00:36,515 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 11f6fecb00005c6636e7ea541ff70dfb/C in TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb. 2024-11-20T09:00:36,515 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/C/b463a22430364744b162d403b2d380a7, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/C/b423d82390304e08a6b1ca9fc8eecdb0, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/C/b3162f2c44a14e33a260e7c69d9c0c71, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/C/83d7f187dcb54ec78e6ec93f80690f58] into tmpdir=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp, totalSize=47.0 K 2024-11-20T09:00:36,515 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.Compactor(225): Compacting b463a22430364744b162d403b2d380a7, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1732093232244 2024-11-20T09:00:36,515 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.Compactor(225): Compacting b423d82390304e08a6b1ca9fc8eecdb0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=66, earliestPutTs=1732093233293 2024-11-20T09:00:36,516 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.Compactor(225): Compacting b3162f2c44a14e33a260e7c69d9c0c71, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=90, earliestPutTs=1732093233388 2024-11-20T09:00:36,517 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.Compactor(225): Compacting 83d7f187dcb54ec78e6ec93f80690f58, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=107, earliestPutTs=1732093234066 2024-11-20T09:00:36,518 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/B/a4693a46097b44c2a65f205f76d6f475 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/B/a4693a46097b44c2a65f205f76d6f475 2024-11-20T09:00:36,521 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:36,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58510 deadline: 1732093296517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:36,522 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:36,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58468 deadline: 1732093296517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:36,522 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:36,522 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:36,522 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:36,522 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:36,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58448 deadline: 1732093296517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:36,523 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:00:36,522 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:36,523 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:00:36,523 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:36,523 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:36,523 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:00:36,524 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 11f6fecb00005c6636e7ea541ff70dfb/B of 11f6fecb00005c6636e7ea541ff70dfb into a4693a46097b44c2a65f205f76d6f475(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T09:00:36,524 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 11f6fecb00005c6636e7ea541ff70dfb: 2024-11-20T09:00:36,524 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., storeName=11f6fecb00005c6636e7ea541ff70dfb/B, priority=12, startTime=1732093236063; duration=0sec 2024-11-20T09:00:36,524 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T09:00:36,524 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 11f6fecb00005c6636e7ea541ff70dfb:B 2024-11-20T09:00:36,530 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 11f6fecb00005c6636e7ea541ff70dfb#C#compaction#256 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T09:00:36,531 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/C/7e28c302b3814e61b87aa454c218dc1c is 50, key is test_row_0/C:col10/1732093235209/Put/seqid=0 2024-11-20T09:00:36,538 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742132_1308 (size=12241) 2024-11-20T09:00:36,544 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/C/7e28c302b3814e61b87aa454c218dc1c as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/C/7e28c302b3814e61b87aa454c218dc1c 2024-11-20T09:00:36,550 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 11f6fecb00005c6636e7ea541ff70dfb/C of 11f6fecb00005c6636e7ea541ff70dfb into 7e28c302b3814e61b87aa454c218dc1c(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T09:00:36,550 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 11f6fecb00005c6636e7ea541ff70dfb: 2024-11-20T09:00:36,550 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., storeName=11f6fecb00005c6636e7ea541ff70dfb/C, priority=12, startTime=1732093236063; duration=0sec 2024-11-20T09:00:36,550 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T09:00:36,550 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 11f6fecb00005c6636e7ea541ff70dfb:C 2024-11-20T09:00:36,729 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:36,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58510 deadline: 1732093296727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:36,730 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:36,730 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:36,730 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:36,730 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:00:36,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58468 deadline: 1732093296727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:36,730 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:36,730 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:36,730 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:36,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58448 deadline: 1732093296727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:36,730 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:00:36,730 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:36,731 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:36,731 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:00:36,895 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=126 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/B/10c856ff67864213877f268d3471a45d 2024-11-20T09:00:36,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/C/d87ce43d0b734a4480a03c4a97fd5d0e is 50, key is test_row_0/C:col10/1732093235228/Put/seqid=0 2024-11-20T09:00:36,907 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742133_1309 (size=12001) 2024-11-20T09:00:37,040 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:37,040 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:37,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58510 deadline: 1732093297037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:37,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58468 deadline: 1732093297037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:37,040 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:37,040 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:37,040 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:37,040 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:37,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58448 deadline: 1732093297037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:37,040 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:37,041 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:00:37,041 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:00:37,041 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:37,041 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:37,041 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:00:37,308 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=126 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/C/d87ce43d0b734a4480a03c4a97fd5d0e 2024-11-20T09:00:37,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/A/aa33f71a11f44f7f925287662e82ba4f as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/A/aa33f71a11f44f7f925287662e82ba4f 2024-11-20T09:00:37,317 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/A/aa33f71a11f44f7f925287662e82ba4f, entries=150, sequenceid=126, filesize=11.7 K 2024-11-20T09:00:37,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/B/10c856ff67864213877f268d3471a45d as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/B/10c856ff67864213877f268d3471a45d 2024-11-20T09:00:37,322 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/B/10c856ff67864213877f268d3471a45d, entries=150, sequenceid=126, filesize=11.7 K 2024-11-20T09:00:37,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/C/d87ce43d0b734a4480a03c4a97fd5d0e as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/C/d87ce43d0b734a4480a03c4a97fd5d0e 2024-11-20T09:00:37,327 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/C/d87ce43d0b734a4480a03c4a97fd5d0e, entries=150, sequenceid=126, filesize=11.7 K 2024-11-20T09:00:37,327 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=93.93 KB/96180 for 11f6fecb00005c6636e7ea541ff70dfb in 1262ms, sequenceid=126, compaction requested=false 2024-11-20T09:00:37,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2603): Flush status journal for 11f6fecb00005c6636e7ea541ff70dfb: 2024-11-20T09:00:37,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb. 2024-11-20T09:00:37,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=65 2024-11-20T09:00:37,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.HMaster(4169): Remote procedure done, pid=65 2024-11-20T09:00:37,331 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=64 2024-11-20T09:00:37,331 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0290 sec 2024-11-20T09:00:37,333 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees in 2.0350 sec 2024-11-20T09:00:37,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-20T09:00:37,447 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T09:00:37,448 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T09:00:37,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] procedure2.ProcedureExecutor(1139): Stored pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees 2024-11-20T09:00:37,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-20T09:00:37,450 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T09:00:37,451 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T09:00:37,451 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T09:00:37,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(8855): Flush requested on 11f6fecb00005c6636e7ea541ff70dfb 2024-11-20T09:00:37,549 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T09:00:37,550 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 11f6fecb00005c6636e7ea541ff70dfb 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-20T09:00:37,555 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/A/1440badfcf3445b0b61e9d33ab2e6d07 is 50, key is test_row_0/A:col10/1732093237549/Put/seqid=0 2024-11-20T09:00:37,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-20T09:00:37,561 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742134_1310 (size=16931) 2024-11-20T09:00:37,562 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=149 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/A/1440badfcf3445b0b61e9d33ab2e6d07 2024-11-20T09:00:37,572 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/B/e69d5c8cfe76497eaceff71cc78851f5 is 50, key is test_row_0/B:col10/1732093237549/Put/seqid=0 2024-11-20T09:00:37,576 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742135_1311 (size=12151) 2024-11-20T09:00:37,576 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:37,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58448 deadline: 1732093297571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:37,576 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:37,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58468 deadline: 1732093297572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:37,576 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:37,576 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:37,576 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:00:37,577 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:37,577 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:37,577 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=149 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/B/e69d5c8cfe76497eaceff71cc78851f5 2024-11-20T09:00:37,577 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:00:37,581 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:37,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58510 deadline: 1732093297576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:37,582 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:37,582 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:37,582 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:00:37,588 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/C/f7ee7238225e4aa5a0e87ed4cf214210 is 50, key is test_row_0/C:col10/1732093237549/Put/seqid=0 2024-11-20T09:00:37,592 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742136_1312 (size=12151) 2024-11-20T09:00:37,593 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=149 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/C/f7ee7238225e4aa5a0e87ed4cf214210 2024-11-20T09:00:37,598 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/A/1440badfcf3445b0b61e9d33ab2e6d07 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/A/1440badfcf3445b0b61e9d33ab2e6d07 2024-11-20T09:00:37,603 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/A/1440badfcf3445b0b61e9d33ab2e6d07, entries=250, sequenceid=149, filesize=16.5 K 2024-11-20T09:00:37,603 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/B/e69d5c8cfe76497eaceff71cc78851f5 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/B/e69d5c8cfe76497eaceff71cc78851f5 2024-11-20T09:00:37,604 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-20T09:00:37,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb. 2024-11-20T09:00:37,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb. as already flushing 2024-11-20T09:00:37,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb. 2024-11-20T09:00:37,604 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 11f6fecb00005c6636e7ea541ff70dfb, NAME => 'TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:37,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 11f6fecb00005c6636e7ea541ff70dfb, NAME => 'TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:37,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 11f6fecb00005c6636e7ea541ff70dfb, NAME => 'TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 11f6fecb00005c6636e7ea541ff70dfb, NAME => 'TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:37,608 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/B/e69d5c8cfe76497eaceff71cc78851f5, entries=150, sequenceid=149, filesize=11.9 K 2024-11-20T09:00:37,608 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/C/f7ee7238225e4aa5a0e87ed4cf214210 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/C/f7ee7238225e4aa5a0e87ed4cf214210 2024-11-20T09:00:37,612 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/C/f7ee7238225e4aa5a0e87ed4cf214210, entries=150, sequenceid=149, filesize=11.9 K 2024-11-20T09:00:37,613 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for 11f6fecb00005c6636e7ea541ff70dfb in 64ms, sequenceid=149, compaction requested=true 2024-11-20T09:00:37,613 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 11f6fecb00005c6636e7ea541ff70dfb: 2024-11-20T09:00:37,613 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 11f6fecb00005c6636e7ea541ff70dfb:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T09:00:37,613 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T09:00:37,613 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T09:00:37,613 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 11f6fecb00005c6636e7ea541ff70dfb:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T09:00:37,614 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T09:00:37,614 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 11f6fecb00005c6636e7ea541ff70dfb:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T09:00:37,614 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T09:00:37,614 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T09:00:37,619 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 41173 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T09:00:37,619 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36393 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T09:00:37,619 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HStore(1541): 11f6fecb00005c6636e7ea541ff70dfb/A is initiating minor compaction (all files) 2024-11-20T09:00:37,619 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1541): 11f6fecb00005c6636e7ea541ff70dfb/B is initiating minor compaction (all files) 2024-11-20T09:00:37,619 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 11f6fecb00005c6636e7ea541ff70dfb/A in TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb. 2024-11-20T09:00:37,619 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 11f6fecb00005c6636e7ea541ff70dfb/B in TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb. 2024-11-20T09:00:37,619 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/A/8846273420cc4c68bd0ecddfafe4fc5d, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/A/aa33f71a11f44f7f925287662e82ba4f, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/A/1440badfcf3445b0b61e9d33ab2e6d07] into tmpdir=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp, totalSize=40.2 K 2024-11-20T09:00:37,619 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/B/a4693a46097b44c2a65f205f76d6f475, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/B/10c856ff67864213877f268d3471a45d, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/B/e69d5c8cfe76497eaceff71cc78851f5] into tmpdir=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp, totalSize=35.5 K 2024-11-20T09:00:37,620 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8846273420cc4c68bd0ecddfafe4fc5d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=107, earliestPutTs=1732093234066 2024-11-20T09:00:37,620 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting a4693a46097b44c2a65f205f76d6f475, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=107, earliestPutTs=1732093234066 2024-11-20T09:00:37,620 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.Compactor(225): Compacting aa33f71a11f44f7f925287662e82ba4f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=126, earliestPutTs=1732093235228 2024-11-20T09:00:37,620 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting 10c856ff67864213877f268d3471a45d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=126, earliestPutTs=1732093235228 2024-11-20T09:00:37,621 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1440badfcf3445b0b61e9d33ab2e6d07, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=149, earliestPutTs=1732093236397 2024-11-20T09:00:37,621 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting e69d5c8cfe76497eaceff71cc78851f5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=149, earliestPutTs=1732093237548 2024-11-20T09:00:37,634 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 11f6fecb00005c6636e7ea541ff70dfb#B#compaction#261 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T09:00:37,634 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/B/f861033725bf4d5091d5919c59a2c43a is 50, key is test_row_0/B:col10/1732093237549/Put/seqid=0 2024-11-20T09:00:37,642 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 11f6fecb00005c6636e7ea541ff70dfb#A#compaction#262 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T09:00:37,642 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/A/1a19596e89ba4a688090ddbdb0d02951 is 50, key is test_row_0/A:col10/1732093237549/Put/seqid=0 2024-11-20T09:00:37,650 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742138_1314 (size=12493) 2024-11-20T09:00:37,658 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742137_1313 (size=12493) 2024-11-20T09:00:37,664 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/B/f861033725bf4d5091d5919c59a2c43a as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/B/f861033725bf4d5091d5919c59a2c43a 2024-11-20T09:00:37,669 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 11f6fecb00005c6636e7ea541ff70dfb/B of 11f6fecb00005c6636e7ea541ff70dfb into f861033725bf4d5091d5919c59a2c43a(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T09:00:37,669 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 11f6fecb00005c6636e7ea541ff70dfb: 2024-11-20T09:00:37,669 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., storeName=11f6fecb00005c6636e7ea541ff70dfb/B, priority=13, startTime=1732093237613; duration=0sec 2024-11-20T09:00:37,670 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T09:00:37,670 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 11f6fecb00005c6636e7ea541ff70dfb:B 2024-11-20T09:00:37,670 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T09:00:37,671 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36393 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T09:00:37,671 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1541): 11f6fecb00005c6636e7ea541ff70dfb/C is initiating minor compaction (all files) 2024-11-20T09:00:37,671 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 11f6fecb00005c6636e7ea541ff70dfb/C in TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb. 2024-11-20T09:00:37,671 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/C/7e28c302b3814e61b87aa454c218dc1c, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/C/d87ce43d0b734a4480a03c4a97fd5d0e, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/C/f7ee7238225e4aa5a0e87ed4cf214210] into tmpdir=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp, totalSize=35.5 K 2024-11-20T09:00:37,672 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting 7e28c302b3814e61b87aa454c218dc1c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=107, earliestPutTs=1732093234066 2024-11-20T09:00:37,672 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting d87ce43d0b734a4480a03c4a97fd5d0e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=126, earliestPutTs=1732093235228 2024-11-20T09:00:37,672 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting f7ee7238225e4aa5a0e87ed4cf214210, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=149, earliestPutTs=1732093237548 2024-11-20T09:00:37,681 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 11f6fecb00005c6636e7ea541ff70dfb#C#compaction#263 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T09:00:37,682 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/C/56bead7fe4f54643b4ad9bc6fd45b21d is 50, key is test_row_0/C:col10/1732093237549/Put/seqid=0 2024-11-20T09:00:37,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(8855): Flush requested on 11f6fecb00005c6636e7ea541ff70dfb 2024-11-20T09:00:37,692 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T09:00:37,692 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 11f6fecb00005c6636e7ea541ff70dfb 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-20T09:00:37,710 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/A/48d9cb2dc3194b68bfed2766b550c0c4 is 50, key is test_row_0/A:col10/1732093237574/Put/seqid=0 2024-11-20T09:00:37,711 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742139_1315 (size=12493) 2024-11-20T09:00:37,718 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742140_1316 (size=14541) 2024-11-20T09:00:37,726 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:37,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58468 deadline: 1732093297719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:37,726 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:37,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58448 deadline: 1732093297720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:37,726 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:37,726 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:37,727 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:00:37,727 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:37,727 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:37,727 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:00:37,727 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:37,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58510 deadline: 1732093297726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:37,728 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:37,728 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:37,728 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:00:37,757 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-20T09:00:37,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb. 2024-11-20T09:00:37,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb. as already flushing 2024-11-20T09:00:37,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb. 2024-11-20T09:00:37,758 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 11f6fecb00005c6636e7ea541ff70dfb, NAME => 'TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:37,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 11f6fecb00005c6636e7ea541ff70dfb, NAME => 'TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:37,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 11f6fecb00005c6636e7ea541ff70dfb, NAME => 'TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 11f6fecb00005c6636e7ea541ff70dfb, NAME => 'TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:37,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-20T09:00:37,841 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:37,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58468 deadline: 1732093297837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:37,841 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:37,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58448 deadline: 1732093297837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:37,841 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:37,841 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:37,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58510 deadline: 1732093297837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:37,841 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:37,841 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:00:37,855 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:37,855 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:37,855 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:00:37,855 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:37,855 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:37,855 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:00:37,911 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-20T09:00:37,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb. 2024-11-20T09:00:37,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb. as already flushing 2024-11-20T09:00:37,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb. 2024-11-20T09:00:37,912 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 11f6fecb00005c6636e7ea541ff70dfb, NAME => 'TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:37,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 11f6fecb00005c6636e7ea541ff70dfb, NAME => 'TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:37,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 11f6fecb00005c6636e7ea541ff70dfb, NAME => 'TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 11f6fecb00005c6636e7ea541ff70dfb, NAME => 'TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:38,049 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:38,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58468 deadline: 1732093298046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:38,050 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:38,050 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:38,050 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:00:38,056 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/A/1a19596e89ba4a688090ddbdb0d02951 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/A/1a19596e89ba4a688090ddbdb0d02951 2024-11-20T09:00:38,061 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 11f6fecb00005c6636e7ea541ff70dfb/A of 11f6fecb00005c6636e7ea541ff70dfb into 1a19596e89ba4a688090ddbdb0d02951(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T09:00:38,061 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 11f6fecb00005c6636e7ea541ff70dfb: 2024-11-20T09:00:38,061 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., storeName=11f6fecb00005c6636e7ea541ff70dfb/A, priority=13, startTime=1732093237613; duration=0sec 2024-11-20T09:00:38,061 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T09:00:38,061 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 11f6fecb00005c6636e7ea541ff70dfb:A 2024-11-20T09:00:38,061 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:38,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58448 deadline: 1732093298057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:38,061 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:38,061 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:38,062 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:00:38,068 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-20T09:00:38,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb. 2024-11-20T09:00:38,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb. as already flushing 2024-11-20T09:00:38,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb. 2024-11-20T09:00:38,069 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 11f6fecb00005c6636e7ea541ff70dfb, NAME => 'TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:38,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 11f6fecb00005c6636e7ea541ff70dfb, NAME => 'TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:38,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 11f6fecb00005c6636e7ea541ff70dfb, NAME => 'TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 11f6fecb00005c6636e7ea541ff70dfb, NAME => 'TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:38,070 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:38,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58510 deadline: 1732093298067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:38,071 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:38,071 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:38,071 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:00:38,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-20T09:00:38,116 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/C/56bead7fe4f54643b4ad9bc6fd45b21d as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/C/56bead7fe4f54643b4ad9bc6fd45b21d 2024-11-20T09:00:38,118 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/A/48d9cb2dc3194b68bfed2766b550c0c4 2024-11-20T09:00:38,121 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 11f6fecb00005c6636e7ea541ff70dfb/C of 11f6fecb00005c6636e7ea541ff70dfb into 56bead7fe4f54643b4ad9bc6fd45b21d(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T09:00:38,121 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 11f6fecb00005c6636e7ea541ff70dfb: 2024-11-20T09:00:38,121 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., storeName=11f6fecb00005c6636e7ea541ff70dfb/C, priority=13, startTime=1732093237614; duration=0sec 2024-11-20T09:00:38,121 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T09:00:38,121 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 11f6fecb00005c6636e7ea541ff70dfb:C 2024-11-20T09:00:38,126 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/B/9fc341a9aef846c1817248db10621a73 is 50, key is test_row_0/B:col10/1732093237574/Put/seqid=0 2024-11-20T09:00:38,130 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742141_1317 (size=12151) 2024-11-20T09:00:38,222 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-20T09:00:38,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb. 2024-11-20T09:00:38,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb. as already flushing 2024-11-20T09:00:38,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb. 2024-11-20T09:00:38,222 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 11f6fecb00005c6636e7ea541ff70dfb, NAME => 'TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:38,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 11f6fecb00005c6636e7ea541ff70dfb, NAME => 'TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:38,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 11f6fecb00005c6636e7ea541ff70dfb, NAME => 'TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 11f6fecb00005c6636e7ea541ff70dfb, NAME => 'TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:38,360 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:38,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58468 deadline: 1732093298357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:38,361 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:38,361 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:38,361 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:00:38,369 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:38,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58448 deadline: 1732093298366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:38,370 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:38,370 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:38,370 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:00:38,375 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-20T09:00:38,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb. 2024-11-20T09:00:38,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb. as already flushing 2024-11-20T09:00:38,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb. 2024-11-20T09:00:38,375 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 11f6fecb00005c6636e7ea541ff70dfb, NAME => 'TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:38,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 11f6fecb00005c6636e7ea541ff70dfb, NAME => 'TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:38,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 11f6fecb00005c6636e7ea541ff70dfb, NAME => 'TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 11f6fecb00005c6636e7ea541ff70dfb, NAME => 'TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:38,380 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:38,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58510 deadline: 1732093298377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:38,381 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:38,381 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:38,381 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:00:38,528 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-20T09:00:38,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb. 2024-11-20T09:00:38,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb. as already flushing 2024-11-20T09:00:38,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb. 2024-11-20T09:00:38,529 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 11f6fecb00005c6636e7ea541ff70dfb, NAME => 'TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:38,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 11f6fecb00005c6636e7ea541ff70dfb, NAME => 'TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:38,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 11f6fecb00005c6636e7ea541ff70dfb, NAME => 'TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 11f6fecb00005c6636e7ea541ff70dfb, NAME => 'TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:38,531 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/B/9fc341a9aef846c1817248db10621a73 2024-11-20T09:00:38,539 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/C/d97e784a8add44859c070d9720afd60d is 50, key is test_row_0/C:col10/1732093237574/Put/seqid=0 2024-11-20T09:00:38,542 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742142_1318 (size=12151) 2024-11-20T09:00:38,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-20T09:00:38,682 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-20T09:00:38,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb. 2024-11-20T09:00:38,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb. as already flushing 2024-11-20T09:00:38,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb. 2024-11-20T09:00:38,682 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 11f6fecb00005c6636e7ea541ff70dfb, NAME => 'TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:38,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 11f6fecb00005c6636e7ea541ff70dfb, NAME => 'TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:38,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 11f6fecb00005c6636e7ea541ff70dfb, NAME => 'TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 11f6fecb00005c6636e7ea541ff70dfb, NAME => 'TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:38,837 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-20T09:00:38,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb. 2024-11-20T09:00:38,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb. as already flushing 2024-11-20T09:00:38,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb. 2024-11-20T09:00:38,838 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 11f6fecb00005c6636e7ea541ff70dfb, NAME => 'TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:38,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 11f6fecb00005c6636e7ea541ff70dfb, NAME => 'TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:38,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 11f6fecb00005c6636e7ea541ff70dfb, NAME => 'TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 11f6fecb00005c6636e7ea541ff70dfb, NAME => 'TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:38,870 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:38,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58468 deadline: 1732093298866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:38,870 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:38,870 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:38,870 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:00:38,878 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:38,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58448 deadline: 1732093298877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:38,878 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:38,878 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:38,878 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:00:38,890 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:38,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58510 deadline: 1732093298887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:38,890 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:38,890 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:38,890 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:00:38,943 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/C/d97e784a8add44859c070d9720afd60d 2024-11-20T09:00:38,949 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/A/48d9cb2dc3194b68bfed2766b550c0c4 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/A/48d9cb2dc3194b68bfed2766b550c0c4 2024-11-20T09:00:38,954 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/A/48d9cb2dc3194b68bfed2766b550c0c4, entries=200, sequenceid=169, filesize=14.2 K 2024-11-20T09:00:38,955 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/B/9fc341a9aef846c1817248db10621a73 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/B/9fc341a9aef846c1817248db10621a73 2024-11-20T09:00:38,959 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/B/9fc341a9aef846c1817248db10621a73, entries=150, sequenceid=169, filesize=11.9 K 2024-11-20T09:00:38,963 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/C/d97e784a8add44859c070d9720afd60d as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/C/d97e784a8add44859c070d9720afd60d 2024-11-20T09:00:38,967 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/C/d97e784a8add44859c070d9720afd60d, entries=150, sequenceid=169, filesize=11.9 K 2024-11-20T09:00:38,968 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=93.93 KB/96180 for 11f6fecb00005c6636e7ea541ff70dfb in 1276ms, sequenceid=169, compaction requested=false 2024-11-20T09:00:38,968 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 11f6fecb00005c6636e7ea541ff70dfb: 2024-11-20T09:00:38,990 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-20T09:00:38,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb. 2024-11-20T09:00:38,991 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2902): Flushing 11f6fecb00005c6636e7ea541ff70dfb 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-20T09:00:38,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/A/78bd4f6bb0254ae586740084a776407f is 50, key is test_row_0/A:col10/1732093237723/Put/seqid=0 2024-11-20T09:00:39,003 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742143_1319 (size=12151) 2024-11-20T09:00:39,004 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=188 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/A/78bd4f6bb0254ae586740084a776407f 2024-11-20T09:00:39,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/B/96ee158b8ad54cbfa83fed3c3e554602 is 50, key is test_row_0/B:col10/1732093237723/Put/seqid=0 2024-11-20T09:00:39,058 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742144_1320 (size=12151) 2024-11-20T09:00:39,058 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=188 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/B/96ee158b8ad54cbfa83fed3c3e554602 2024-11-20T09:00:39,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/C/0c582d7f578b4e83bae3ac21f218d268 is 50, key is test_row_0/C:col10/1732093237723/Put/seqid=0 2024-11-20T09:00:39,076 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742145_1321 (size=12151) 2024-11-20T09:00:39,078 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=188 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/C/0c582d7f578b4e83bae3ac21f218d268 2024-11-20T09:00:39,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/A/78bd4f6bb0254ae586740084a776407f as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/A/78bd4f6bb0254ae586740084a776407f 2024-11-20T09:00:39,091 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/A/78bd4f6bb0254ae586740084a776407f, entries=150, sequenceid=188, filesize=11.9 K 2024-11-20T09:00:39,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/B/96ee158b8ad54cbfa83fed3c3e554602 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/B/96ee158b8ad54cbfa83fed3c3e554602 2024-11-20T09:00:39,097 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/B/96ee158b8ad54cbfa83fed3c3e554602, entries=150, sequenceid=188, filesize=11.9 K 2024-11-20T09:00:39,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/C/0c582d7f578b4e83bae3ac21f218d268 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/C/0c582d7f578b4e83bae3ac21f218d268 2024-11-20T09:00:39,110 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/C/0c582d7f578b4e83bae3ac21f218d268, entries=150, sequenceid=188, filesize=11.9 K 2024-11-20T09:00:39,111 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=0 B/0 for 11f6fecb00005c6636e7ea541ff70dfb in 120ms, sequenceid=188, compaction requested=true 2024-11-20T09:00:39,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2603): Flush status journal for 11f6fecb00005c6636e7ea541ff70dfb: 2024-11-20T09:00:39,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb. 2024-11-20T09:00:39,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=67 2024-11-20T09:00:39,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.HMaster(4169): Remote procedure done, pid=67 2024-11-20T09:00:39,116 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=67, resume processing ppid=66 2024-11-20T09:00:39,116 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, ppid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6620 sec 2024-11-20T09:00:39,119 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees in 1.6690 sec 2024-11-20T09:00:39,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-20T09:00:39,597 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T09:00:39,598 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T09:00:39,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] procedure2.ProcedureExecutor(1139): Stored pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees 2024-11-20T09:00:39,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-20T09:00:39,601 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T09:00:39,601 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T09:00:39,602 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T09:00:39,645 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-20T09:00:39,645 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-20T09:00:39,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-20T09:00:39,757 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-20T09:00:39,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb. 2024-11-20T09:00:39,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2603): Flush status journal for 11f6fecb00005c6636e7ea541ff70dfb: 2024-11-20T09:00:39,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb. 2024-11-20T09:00:39,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=69 2024-11-20T09:00:39,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.HMaster(4169): Remote procedure done, pid=69 2024-11-20T09:00:39,761 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=69, resume processing ppid=68 2024-11-20T09:00:39,761 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 158 msec 2024-11-20T09:00:39,763 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees in 164 msec 2024-11-20T09:00:39,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(8855): Flush requested on 11f6fecb00005c6636e7ea541ff70dfb 2024-11-20T09:00:39,895 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T09:00:39,895 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 11f6fecb00005c6636e7ea541ff70dfb 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-20T09:00:39,902 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/A/7b478c09b1004a45a5f4d2d89596c235 is 50, key is test_row_0/A:col10/1732093239894/Put/seqid=0 2024-11-20T09:00:39,916 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742146_1322 (size=14541) 2024-11-20T09:00:39,916 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=201 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/A/7b478c09b1004a45a5f4d2d89596c235 2024-11-20T09:00:39,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-20T09:00:39,917 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T09:00:39,918 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T09:00:39,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] procedure2.ProcedureExecutor(1139): Stored pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees 2024-11-20T09:00:39,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-20T09:00:39,920 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T09:00:39,921 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T09:00:39,921 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T09:00:39,928 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/B/7968deabefaa43908a63ff4a07bc04c4 is 50, key is test_row_0/B:col10/1732093239894/Put/seqid=0 2024-11-20T09:00:39,932 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742147_1323 (size=12151) 2024-11-20T09:00:39,951 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:39,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58510 deadline: 1732093299946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:39,952 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:39,952 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:39,952 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:00:39,957 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:39,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58448 deadline: 1732093299951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:39,957 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:39,957 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:39,957 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:39,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58468 deadline: 1732093299952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:39,957 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:00:39,957 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:39,957 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:39,957 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:00:40,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-20T09:00:40,061 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:40,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58510 deadline: 1732093300057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:40,062 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:40,062 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:40,062 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:00:40,069 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:40,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58468 deadline: 1732093300067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:40,070 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:40,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58448 deadline: 1732093300067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:40,070 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:40,070 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:40,070 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:00:40,071 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:40,071 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:40,071 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:00:40,074 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-20T09:00:40,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb. 2024-11-20T09:00:40,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb. as already flushing 2024-11-20T09:00:40,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb. 2024-11-20T09:00:40,074 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 11f6fecb00005c6636e7ea541ff70dfb, NAME => 'TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:40,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 11f6fecb00005c6636e7ea541ff70dfb, NAME => 'TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:40,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 11f6fecb00005c6636e7ea541ff70dfb, NAME => 'TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 11f6fecb00005c6636e7ea541ff70dfb, NAME => 'TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:40,227 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-20T09:00:40,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb. 2024-11-20T09:00:40,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb. as already flushing 2024-11-20T09:00:40,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb. 2024-11-20T09:00:40,228 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 11f6fecb00005c6636e7ea541ff70dfb, NAME => 'TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:40,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 11f6fecb00005c6636e7ea541ff70dfb, NAME => 'TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:40,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 11f6fecb00005c6636e7ea541ff70dfb, NAME => 'TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 11f6fecb00005c6636e7ea541ff70dfb, NAME => 'TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:40,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-20T09:00:40,270 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:40,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58510 deadline: 1732093300267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:40,271 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:40,271 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:40,271 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:00:40,280 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:40,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58468 deadline: 1732093300276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:40,281 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:40,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58448 deadline: 1732093300277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:40,281 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:40,281 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:40,281 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:00:40,281 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:40,281 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:40,281 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:00:40,333 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=201 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/B/7968deabefaa43908a63ff4a07bc04c4 2024-11-20T09:00:40,341 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/C/ab7ee26379784aafa2ac4eb0173d2f2a is 50, key is test_row_0/C:col10/1732093239894/Put/seqid=0 2024-11-20T09:00:40,345 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742148_1324 (size=12151) 2024-11-20T09:00:40,380 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-20T09:00:40,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb. 2024-11-20T09:00:40,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb. as already flushing 2024-11-20T09:00:40,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb. 2024-11-20T09:00:40,381 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 11f6fecb00005c6636e7ea541ff70dfb, NAME => 'TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:40,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 11f6fecb00005c6636e7ea541ff70dfb, NAME => 'TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:40,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 11f6fecb00005c6636e7ea541ff70dfb, NAME => 'TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 11f6fecb00005c6636e7ea541ff70dfb, NAME => 'TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:40,458 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:40,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58454 deadline: 1732093300457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:40,459 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to be1eb2620e0e,34223,1732093159960 for 'test_row_0' in 11f6fecb00005c6636e7ea541ff70dfb of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8207 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-20T09:00:40,459 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:40,459 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:40,460 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:00:40,470 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:40,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58452 deadline: 1732093300467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:40,471 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to be1eb2620e0e,34223,1732093159960 for 'test_row_2' in 11f6fecb00005c6636e7ea541ff70dfb of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8218 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-20T09:00:40,471 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:40,471 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:40,471 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:00:40,534 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-20T09:00:40,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb. 2024-11-20T09:00:40,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb. as already flushing 2024-11-20T09:00:40,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb. 2024-11-20T09:00:40,534 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 11f6fecb00005c6636e7ea541ff70dfb, NAME => 'TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:40,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 11f6fecb00005c6636e7ea541ff70dfb, NAME => 'TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:40,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 11f6fecb00005c6636e7ea541ff70dfb, NAME => 'TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 11f6fecb00005c6636e7ea541ff70dfb, NAME => 'TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:40,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-20T09:00:40,580 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:40,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58510 deadline: 1732093300577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:40,580 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:40,580 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:40,580 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:00:40,589 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:40,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58468 deadline: 1732093300586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:40,589 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:40,589 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:40,589 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:40,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58448 deadline: 1732093300586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:40,589 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:00:40,590 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:40,590 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:40,590 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:00:40,687 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-20T09:00:40,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb. 2024-11-20T09:00:40,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb. as already flushing 2024-11-20T09:00:40,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb. 2024-11-20T09:00:40,687 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 11f6fecb00005c6636e7ea541ff70dfb, NAME => 'TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:40,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 11f6fecb00005c6636e7ea541ff70dfb, NAME => 'TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:40,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 11f6fecb00005c6636e7ea541ff70dfb, NAME => 'TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 11f6fecb00005c6636e7ea541ff70dfb, NAME => 'TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:40,745 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=201 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/C/ab7ee26379784aafa2ac4eb0173d2f2a 2024-11-20T09:00:40,750 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/A/7b478c09b1004a45a5f4d2d89596c235 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/A/7b478c09b1004a45a5f4d2d89596c235 2024-11-20T09:00:40,754 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/A/7b478c09b1004a45a5f4d2d89596c235, entries=200, sequenceid=201, filesize=14.2 K 2024-11-20T09:00:40,755 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/B/7968deabefaa43908a63ff4a07bc04c4 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/B/7968deabefaa43908a63ff4a07bc04c4 2024-11-20T09:00:40,758 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/B/7968deabefaa43908a63ff4a07bc04c4, entries=150, sequenceid=201, filesize=11.9 K 2024-11-20T09:00:40,759 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/C/ab7ee26379784aafa2ac4eb0173d2f2a as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/C/ab7ee26379784aafa2ac4eb0173d2f2a 2024-11-20T09:00:40,763 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/C/ab7ee26379784aafa2ac4eb0173d2f2a, entries=150, sequenceid=201, filesize=11.9 K 2024-11-20T09:00:40,764 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 11f6fecb00005c6636e7ea541ff70dfb in 869ms, sequenceid=201, compaction requested=true 2024-11-20T09:00:40,764 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 11f6fecb00005c6636e7ea541ff70dfb: 2024-11-20T09:00:40,764 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 11f6fecb00005c6636e7ea541ff70dfb:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T09:00:40,764 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T09:00:40,764 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 11f6fecb00005c6636e7ea541ff70dfb:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T09:00:40,764 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T09:00:40,764 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 11f6fecb00005c6636e7ea541ff70dfb:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T09:00:40,764 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T09:00:40,764 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T09:00:40,764 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T09:00:40,765 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48946 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T09:00:40,765 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 53726 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T09:00:40,765 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1541): 11f6fecb00005c6636e7ea541ff70dfb/B is initiating minor compaction (all files) 2024-11-20T09:00:40,765 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HStore(1541): 11f6fecb00005c6636e7ea541ff70dfb/A is initiating minor compaction (all files) 2024-11-20T09:00:40,766 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 11f6fecb00005c6636e7ea541ff70dfb/B in TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb. 2024-11-20T09:00:40,766 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 11f6fecb00005c6636e7ea541ff70dfb/A in TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb. 2024-11-20T09:00:40,766 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/B/f861033725bf4d5091d5919c59a2c43a, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/B/9fc341a9aef846c1817248db10621a73, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/B/96ee158b8ad54cbfa83fed3c3e554602, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/B/7968deabefaa43908a63ff4a07bc04c4] into tmpdir=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp, totalSize=47.8 K 2024-11-20T09:00:40,766 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/A/1a19596e89ba4a688090ddbdb0d02951, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/A/48d9cb2dc3194b68bfed2766b550c0c4, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/A/78bd4f6bb0254ae586740084a776407f, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/A/7b478c09b1004a45a5f4d2d89596c235] into tmpdir=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp, totalSize=52.5 K 2024-11-20T09:00:40,766 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting f861033725bf4d5091d5919c59a2c43a, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=149, earliestPutTs=1732093237548 2024-11-20T09:00:40,766 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1a19596e89ba4a688090ddbdb0d02951, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=149, earliestPutTs=1732093237548 2024-11-20T09:00:40,767 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting 9fc341a9aef846c1817248db10621a73, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1732093237571 2024-11-20T09:00:40,767 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.Compactor(225): Compacting 48d9cb2dc3194b68bfed2766b550c0c4, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1732093237571 2024-11-20T09:00:40,767 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting 96ee158b8ad54cbfa83fed3c3e554602, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=188, earliestPutTs=1732093237712 2024-11-20T09:00:40,767 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.Compactor(225): Compacting 78bd4f6bb0254ae586740084a776407f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=188, earliestPutTs=1732093237712 2024-11-20T09:00:40,767 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7b478c09b1004a45a5f4d2d89596c235, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=201, earliestPutTs=1732093239882 2024-11-20T09:00:40,767 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting 7968deabefaa43908a63ff4a07bc04c4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=201, earliestPutTs=1732093239890 2024-11-20T09:00:40,776 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 11f6fecb00005c6636e7ea541ff70dfb#B#compaction#273 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-20T09:00:40,777 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/B/8746a2c44aa04cbaaf775c28d8fe006e is 50, key is test_row_0/B:col10/1732093239894/Put/seqid=0 2024-11-20T09:00:40,777 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 11f6fecb00005c6636e7ea541ff70dfb#A#compaction#274 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T09:00:40,778 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/A/5f951ec18f604abeb651fd0e5ec4bf7c is 50, key is test_row_0/A:col10/1732093239894/Put/seqid=0 2024-11-20T09:00:40,782 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742149_1325 (size=12629) 2024-11-20T09:00:40,785 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742150_1326 (size=12629) 2024-11-20T09:00:40,840 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-20T09:00:40,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb. 2024-11-20T09:00:40,840 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2902): Flushing 11f6fecb00005c6636e7ea541ff70dfb 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-20T09:00:40,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/A/411dcbf93be246be96977c2bc8d25cdf is 50, key is test_row_0/A:col10/1732093239950/Put/seqid=0 2024-11-20T09:00:40,849 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742151_1327 (size=12151) 2024-11-20T09:00:41,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-20T09:00:41,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(8855): Flush requested on 11f6fecb00005c6636e7ea541ff70dfb 2024-11-20T09:00:41,090 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb. as already flushing 2024-11-20T09:00:41,111 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:41,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58468 deadline: 1732093301106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:41,112 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:41,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58448 deadline: 1732093301107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:41,112 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:41,112 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:41,112 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:00:41,112 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:41,112 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:41,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58510 deadline: 1732093301108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:41,112 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:41,112 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:00:41,113 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:41,113 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:41,113 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:00:41,187 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/B/8746a2c44aa04cbaaf775c28d8fe006e as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/B/8746a2c44aa04cbaaf775c28d8fe006e 2024-11-20T09:00:41,192 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/A/5f951ec18f604abeb651fd0e5ec4bf7c as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/A/5f951ec18f604abeb651fd0e5ec4bf7c 2024-11-20T09:00:41,193 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 11f6fecb00005c6636e7ea541ff70dfb/B of 11f6fecb00005c6636e7ea541ff70dfb into 8746a2c44aa04cbaaf775c28d8fe006e(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T09:00:41,193 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 11f6fecb00005c6636e7ea541ff70dfb: 2024-11-20T09:00:41,193 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., storeName=11f6fecb00005c6636e7ea541ff70dfb/B, priority=12, startTime=1732093240764; duration=0sec 2024-11-20T09:00:41,193 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T09:00:41,193 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 11f6fecb00005c6636e7ea541ff70dfb:B 2024-11-20T09:00:41,193 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T09:00:41,194 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48946 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T09:00:41,195 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1541): 11f6fecb00005c6636e7ea541ff70dfb/C is initiating minor compaction (all files) 2024-11-20T09:00:41,195 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 11f6fecb00005c6636e7ea541ff70dfb/C in TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb. 2024-11-20T09:00:41,195 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/C/56bead7fe4f54643b4ad9bc6fd45b21d, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/C/d97e784a8add44859c070d9720afd60d, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/C/0c582d7f578b4e83bae3ac21f218d268, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/C/ab7ee26379784aafa2ac4eb0173d2f2a] into tmpdir=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp, totalSize=47.8 K 2024-11-20T09:00:41,196 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting 56bead7fe4f54643b4ad9bc6fd45b21d, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=149, earliestPutTs=1732093237548 2024-11-20T09:00:41,197 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 11f6fecb00005c6636e7ea541ff70dfb/A of 11f6fecb00005c6636e7ea541ff70dfb into 5f951ec18f604abeb651fd0e5ec4bf7c(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T09:00:41,197 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 11f6fecb00005c6636e7ea541ff70dfb: 2024-11-20T09:00:41,197 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting d97e784a8add44859c070d9720afd60d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1732093237571 2024-11-20T09:00:41,197 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., storeName=11f6fecb00005c6636e7ea541ff70dfb/A, priority=12, startTime=1732093240764; duration=0sec 2024-11-20T09:00:41,197 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T09:00:41,197 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 11f6fecb00005c6636e7ea541ff70dfb:A 2024-11-20T09:00:41,197 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting 0c582d7f578b4e83bae3ac21f218d268, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=188, earliestPutTs=1732093237712 2024-11-20T09:00:41,197 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting ab7ee26379784aafa2ac4eb0173d2f2a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=201, earliestPutTs=1732093239890 2024-11-20T09:00:41,205 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 11f6fecb00005c6636e7ea541ff70dfb#C#compaction#276 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T09:00:41,205 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/C/79f06b0b6eff4d8e82a8c9b337f84cfd is 50, key is test_row_0/C:col10/1732093239894/Put/seqid=0 2024-11-20T09:00:41,209 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742152_1328 (size=12629) 2024-11-20T09:00:41,213 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/C/79f06b0b6eff4d8e82a8c9b337f84cfd as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/C/79f06b0b6eff4d8e82a8c9b337f84cfd 2024-11-20T09:00:41,218 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 11f6fecb00005c6636e7ea541ff70dfb/C of 11f6fecb00005c6636e7ea541ff70dfb into 79f06b0b6eff4d8e82a8c9b337f84cfd(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T09:00:41,218 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 11f6fecb00005c6636e7ea541ff70dfb: 2024-11-20T09:00:41,218 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., storeName=11f6fecb00005c6636e7ea541ff70dfb/C, priority=12, startTime=1732093240764; duration=0sec 2024-11-20T09:00:41,218 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T09:00:41,218 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 11f6fecb00005c6636e7ea541ff70dfb:C 2024-11-20T09:00:41,220 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:41,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58468 deadline: 1732093301217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:41,220 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:41,220 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:41,220 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:41,221 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:00:41,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58448 deadline: 1732093301217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:41,221 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:41,221 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:41,221 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:41,221 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:00:41,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58510 deadline: 1732093301217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:41,221 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:41,221 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:41,221 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:00:41,250 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=225 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/A/411dcbf93be246be96977c2bc8d25cdf 2024-11-20T09:00:41,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/B/11b0cbe124f846db8ef755b72e55d758 is 50, key is test_row_0/B:col10/1732093239950/Put/seqid=0 2024-11-20T09:00:41,265 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742153_1329 (size=12151) 2024-11-20T09:00:41,267 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=225 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/B/11b0cbe124f846db8ef755b72e55d758 2024-11-20T09:00:41,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/C/510e463161fc4fc8937b29097c47acd4 is 50, key is test_row_0/C:col10/1732093239950/Put/seqid=0 2024-11-20T09:00:41,282 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742154_1330 (size=12151) 2024-11-20T09:00:41,429 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:41,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58468 deadline: 1732093301427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:41,430 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:41,429 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:41,430 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:41,430 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:00:41,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58448 deadline: 1732093301427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:41,430 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:41,430 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:41,430 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:41,430 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:00:41,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58510 deadline: 1732093301427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:41,430 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:41,430 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:41,430 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:00:41,683 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=225 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/C/510e463161fc4fc8937b29097c47acd4 2024-11-20T09:00:41,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/A/411dcbf93be246be96977c2bc8d25cdf as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/A/411dcbf93be246be96977c2bc8d25cdf 2024-11-20T09:00:41,691 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/A/411dcbf93be246be96977c2bc8d25cdf, entries=150, sequenceid=225, filesize=11.9 K 2024-11-20T09:00:41,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/B/11b0cbe124f846db8ef755b72e55d758 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/B/11b0cbe124f846db8ef755b72e55d758 2024-11-20T09:00:41,696 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/B/11b0cbe124f846db8ef755b72e55d758, entries=150, sequenceid=225, filesize=11.9 K 2024-11-20T09:00:41,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/C/510e463161fc4fc8937b29097c47acd4 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/C/510e463161fc4fc8937b29097c47acd4 2024-11-20T09:00:41,701 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/C/510e463161fc4fc8937b29097c47acd4, entries=150, sequenceid=225, filesize=11.9 K 2024-11-20T09:00:41,702 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 11f6fecb00005c6636e7ea541ff70dfb in 862ms, sequenceid=225, compaction requested=false 2024-11-20T09:00:41,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2603): Flush status journal for 11f6fecb00005c6636e7ea541ff70dfb: 2024-11-20T09:00:41,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb. 2024-11-20T09:00:41,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=71 2024-11-20T09:00:41,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.HMaster(4169): Remote procedure done, pid=71 2024-11-20T09:00:41,706 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=71, resume processing ppid=70 2024-11-20T09:00:41,706 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, ppid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7820 sec 2024-11-20T09:00:41,711 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees in 1.7910 sec 2024-11-20T09:00:41,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(8855): Flush requested on 11f6fecb00005c6636e7ea541ff70dfb 2024-11-20T09:00:41,740 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T09:00:41,740 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 11f6fecb00005c6636e7ea541ff70dfb 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-20T09:00:41,746 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/A/48a62c29d37d4ba09b8f065e302acb5e is 50, key is test_row_0/A:col10/1732093241107/Put/seqid=0 2024-11-20T09:00:41,751 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742155_1331 (size=12151) 2024-11-20T09:00:41,776 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:41,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58468 deadline: 1732093301771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:41,776 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:41,776 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:41,776 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:41,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58448 deadline: 1732093301772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:41,776 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:00:41,777 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:41,777 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:41,777 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:00:41,778 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:41,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58510 deadline: 1732093301775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:41,779 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:41,779 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:41,779 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:00:41,889 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:41,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58468 deadline: 1732093301887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:41,890 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:41,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58448 deadline: 1732093301887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:41,890 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:41,890 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:41,890 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:00:41,890 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:41,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58510 deadline: 1732093301887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:41,890 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:41,890 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:41,890 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:00:41,890 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:41,890 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:41,890 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:00:42,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-20T09:00:42,068 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T09:00:42,068 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T09:00:42,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] procedure2.ProcedureExecutor(1139): Stored pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees 2024-11-20T09:00:42,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-20T09:00:42,070 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T09:00:42,071 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T09:00:42,071 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T09:00:42,100 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:42,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58468 deadline: 1732093302097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:42,101 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:42,101 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:42,101 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:00:42,101 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:42,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58510 deadline: 1732093302097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:42,102 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:42,102 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:42,102 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:42,102 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:00:42,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58448 deadline: 1732093302098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:42,102 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:42,102 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:42,102 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:00:42,152 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=242 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/A/48a62c29d37d4ba09b8f065e302acb5e 2024-11-20T09:00:42,159 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/B/97960baeee5d42998d385d9c3eb18b6c is 50, key is test_row_0/B:col10/1732093241107/Put/seqid=0 2024-11-20T09:00:42,162 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742156_1332 (size=12151) 2024-11-20T09:00:42,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-20T09:00:42,223 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-20T09:00:42,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb. 2024-11-20T09:00:42,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb. as already flushing 2024-11-20T09:00:42,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb. 2024-11-20T09:00:42,224 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 11f6fecb00005c6636e7ea541ff70dfb, NAME => 'TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:42,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 11f6fecb00005c6636e7ea541ff70dfb, NAME => 'TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:42,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 11f6fecb00005c6636e7ea541ff70dfb, NAME => 'TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 11f6fecb00005c6636e7ea541ff70dfb, NAME => 'TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:42,377 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-20T09:00:42,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb. 2024-11-20T09:00:42,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb. as already flushing 2024-11-20T09:00:42,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb. 2024-11-20T09:00:42,377 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 11f6fecb00005c6636e7ea541ff70dfb, NAME => 'TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:42,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 11f6fecb00005c6636e7ea541ff70dfb, NAME => 'TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:42,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 11f6fecb00005c6636e7ea541ff70dfb, NAME => 'TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 11f6fecb00005c6636e7ea541ff70dfb, NAME => 'TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:42,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-20T09:00:42,409 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:42,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58468 deadline: 1732093302407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:42,410 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:42,410 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:42,410 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:42,410 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:00:42,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58510 deadline: 1732093302407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:42,410 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:42,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58448 deadline: 1732093302407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:42,410 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:42,410 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:42,410 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:00:42,410 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:42,410 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:42,410 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:00:42,530 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-20T09:00:42,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb. 2024-11-20T09:00:42,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb. as already flushing 2024-11-20T09:00:42,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb. 2024-11-20T09:00:42,530 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 11f6fecb00005c6636e7ea541ff70dfb, NAME => 'TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:42,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 11f6fecb00005c6636e7ea541ff70dfb, NAME => 'TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:42,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 11f6fecb00005c6636e7ea541ff70dfb, NAME => 'TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 11f6fecb00005c6636e7ea541ff70dfb, NAME => 'TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:42,563 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=242 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/B/97960baeee5d42998d385d9c3eb18b6c 2024-11-20T09:00:42,570 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/C/5ef39d459ba1454eaa6e167b854acf45 is 50, key is test_row_0/C:col10/1732093241107/Put/seqid=0 2024-11-20T09:00:42,574 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742157_1333 (size=12151) 2024-11-20T09:00:42,683 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-20T09:00:42,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb. 2024-11-20T09:00:42,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb. as already flushing 2024-11-20T09:00:42,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb. 2024-11-20T09:00:42,683 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 11f6fecb00005c6636e7ea541ff70dfb, NAME => 'TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:42,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 11f6fecb00005c6636e7ea541ff70dfb, NAME => 'TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:42,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 11f6fecb00005c6636e7ea541ff70dfb, NAME => 'TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 11f6fecb00005c6636e7ea541ff70dfb, NAME => 'TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:42,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-20T09:00:42,836 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-20T09:00:42,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb. 2024-11-20T09:00:42,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb. as already flushing 2024-11-20T09:00:42,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb. 2024-11-20T09:00:42,836 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 11f6fecb00005c6636e7ea541ff70dfb, NAME => 'TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:42,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 11f6fecb00005c6636e7ea541ff70dfb, NAME => 'TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:42,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 11f6fecb00005c6636e7ea541ff70dfb, NAME => 'TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 11f6fecb00005c6636e7ea541ff70dfb, NAME => 'TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:42,919 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:42,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58468 deadline: 1732093302917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:42,919 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:42,920 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:42,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58510 deadline: 1732093302917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:42,920 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:42,920 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:00:42,920 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:42,920 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:42,920 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:42,920 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:00:42,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58448 deadline: 1732093302917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:42,920 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:42,920 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:42,920 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:00:42,975 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=242 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/C/5ef39d459ba1454eaa6e167b854acf45 2024-11-20T09:00:42,980 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/A/48a62c29d37d4ba09b8f065e302acb5e as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/A/48a62c29d37d4ba09b8f065e302acb5e 2024-11-20T09:00:42,984 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/A/48a62c29d37d4ba09b8f065e302acb5e, entries=150, sequenceid=242, filesize=11.9 K 2024-11-20T09:00:42,985 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/B/97960baeee5d42998d385d9c3eb18b6c as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/B/97960baeee5d42998d385d9c3eb18b6c 2024-11-20T09:00:42,989 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-20T09:00:42,989 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/B/97960baeee5d42998d385d9c3eb18b6c, entries=150, sequenceid=242, filesize=11.9 K 2024-11-20T09:00:42,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb. 2024-11-20T09:00:42,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb. as already flushing 2024-11-20T09:00:42,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb. 2024-11-20T09:00:42,990 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 11f6fecb00005c6636e7ea541ff70dfb, NAME => 'TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:42,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 11f6fecb00005c6636e7ea541ff70dfb, NAME => 'TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:42,990 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/C/5ef39d459ba1454eaa6e167b854acf45 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/C/5ef39d459ba1454eaa6e167b854acf45 2024-11-20T09:00:42,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 11f6fecb00005c6636e7ea541ff70dfb, NAME => 'TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 11f6fecb00005c6636e7ea541ff70dfb, NAME => 'TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:42,995 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/C/5ef39d459ba1454eaa6e167b854acf45, entries=150, sequenceid=242, filesize=11.9 K 2024-11-20T09:00:42,996 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 11f6fecb00005c6636e7ea541ff70dfb in 1256ms, sequenceid=242, compaction requested=true 2024-11-20T09:00:42,996 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 11f6fecb00005c6636e7ea541ff70dfb: 2024-11-20T09:00:42,996 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 11f6fecb00005c6636e7ea541ff70dfb:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T09:00:42,996 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T09:00:42,996 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T09:00:42,996 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T09:00:42,996 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 11f6fecb00005c6636e7ea541ff70dfb:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T09:00:42,996 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T09:00:42,996 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 11f6fecb00005c6636e7ea541ff70dfb:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T09:00:42,996 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T09:00:42,997 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36931 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T09:00:42,997 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36931 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T09:00:42,997 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1541): 11f6fecb00005c6636e7ea541ff70dfb/B is initiating minor compaction (all files) 2024-11-20T09:00:42,997 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HStore(1541): 11f6fecb00005c6636e7ea541ff70dfb/A is initiating minor compaction (all files) 2024-11-20T09:00:42,997 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 11f6fecb00005c6636e7ea541ff70dfb/B in TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb. 2024-11-20T09:00:42,997 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 11f6fecb00005c6636e7ea541ff70dfb/A in TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb. 2024-11-20T09:00:42,997 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/B/8746a2c44aa04cbaaf775c28d8fe006e, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/B/11b0cbe124f846db8ef755b72e55d758, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/B/97960baeee5d42998d385d9c3eb18b6c] into tmpdir=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp, totalSize=36.1 K 2024-11-20T09:00:42,997 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/A/5f951ec18f604abeb651fd0e5ec4bf7c, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/A/411dcbf93be246be96977c2bc8d25cdf, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/A/48a62c29d37d4ba09b8f065e302acb5e] into tmpdir=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp, totalSize=36.1 K 2024-11-20T09:00:42,998 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting 8746a2c44aa04cbaaf775c28d8fe006e, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=201, earliestPutTs=1732093239890 2024-11-20T09:00:42,998 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5f951ec18f604abeb651fd0e5ec4bf7c, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=201, earliestPutTs=1732093239890 2024-11-20T09:00:42,998 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.Compactor(225): Compacting 411dcbf93be246be96977c2bc8d25cdf, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1732093239944 2024-11-20T09:00:42,998 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting 11b0cbe124f846db8ef755b72e55d758, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1732093239944 2024-11-20T09:00:42,999 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.Compactor(225): Compacting 48a62c29d37d4ba09b8f065e302acb5e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=242, earliestPutTs=1732093241107 2024-11-20T09:00:42,999 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting 97960baeee5d42998d385d9c3eb18b6c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=242, earliestPutTs=1732093241107 2024-11-20T09:00:43,008 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 11f6fecb00005c6636e7ea541ff70dfb#A#compaction#282 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T09:00:43,009 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/A/1923e25e14774215b677c209bfc5b85e is 50, key is test_row_0/A:col10/1732093241107/Put/seqid=0 2024-11-20T09:00:43,010 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 11f6fecb00005c6636e7ea541ff70dfb#B#compaction#283 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T09:00:43,010 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/B/f97fc93675a141cabe97a8bb7d72cb10 is 50, key is test_row_0/B:col10/1732093241107/Put/seqid=0 2024-11-20T09:00:43,021 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742158_1334 (size=12731) 2024-11-20T09:00:43,030 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742159_1335 (size=12731) 2024-11-20T09:00:43,143 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-20T09:00:43,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb. 2024-11-20T09:00:43,144 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2902): Flushing 11f6fecb00005c6636e7ea541ff70dfb 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-20T09:00:43,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/A/b2d6779930e549af96d69528268996b3 is 50, key is test_row_0/A:col10/1732093241774/Put/seqid=0 2024-11-20T09:00:43,172 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742160_1336 (size=12301) 2024-11-20T09:00:43,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-20T09:00:43,426 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/A/1923e25e14774215b677c209bfc5b85e as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/A/1923e25e14774215b677c209bfc5b85e 2024-11-20T09:00:43,432 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 11f6fecb00005c6636e7ea541ff70dfb/A of 11f6fecb00005c6636e7ea541ff70dfb into 1923e25e14774215b677c209bfc5b85e(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T09:00:43,432 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 11f6fecb00005c6636e7ea541ff70dfb: 2024-11-20T09:00:43,432 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., storeName=11f6fecb00005c6636e7ea541ff70dfb/A, priority=13, startTime=1732093242996; duration=0sec 2024-11-20T09:00:43,432 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T09:00:43,432 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 11f6fecb00005c6636e7ea541ff70dfb:A 2024-11-20T09:00:43,432 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T09:00:43,434 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36931 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T09:00:43,434 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HStore(1541): 11f6fecb00005c6636e7ea541ff70dfb/C is initiating minor compaction (all files) 2024-11-20T09:00:43,434 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 11f6fecb00005c6636e7ea541ff70dfb/C in TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb. 2024-11-20T09:00:43,434 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/C/79f06b0b6eff4d8e82a8c9b337f84cfd, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/C/510e463161fc4fc8937b29097c47acd4, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/C/5ef39d459ba1454eaa6e167b854acf45] into tmpdir=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp, totalSize=36.1 K 2024-11-20T09:00:43,435 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.Compactor(225): Compacting 79f06b0b6eff4d8e82a8c9b337f84cfd, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=201, earliestPutTs=1732093239890 2024-11-20T09:00:43,435 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/B/f97fc93675a141cabe97a8bb7d72cb10 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/B/f97fc93675a141cabe97a8bb7d72cb10 2024-11-20T09:00:43,435 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.Compactor(225): Compacting 510e463161fc4fc8937b29097c47acd4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1732093239944 2024-11-20T09:00:43,436 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5ef39d459ba1454eaa6e167b854acf45, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=242, earliestPutTs=1732093241107 2024-11-20T09:00:43,440 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 11f6fecb00005c6636e7ea541ff70dfb/B of 11f6fecb00005c6636e7ea541ff70dfb into f97fc93675a141cabe97a8bb7d72cb10(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T09:00:43,440 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 11f6fecb00005c6636e7ea541ff70dfb: 2024-11-20T09:00:43,440 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., storeName=11f6fecb00005c6636e7ea541ff70dfb/B, priority=13, startTime=1732093242996; duration=0sec 2024-11-20T09:00:43,440 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T09:00:43,440 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 11f6fecb00005c6636e7ea541ff70dfb:B 2024-11-20T09:00:43,445 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 11f6fecb00005c6636e7ea541ff70dfb#C#compaction#285 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T09:00:43,446 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/C/c38cf285c47348c493ba151b8373d971 is 50, key is test_row_0/C:col10/1732093241107/Put/seqid=0 2024-11-20T09:00:43,451 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742161_1337 (size=12731) 2024-11-20T09:00:43,457 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/C/c38cf285c47348c493ba151b8373d971 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/C/c38cf285c47348c493ba151b8373d971 2024-11-20T09:00:43,461 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 11f6fecb00005c6636e7ea541ff70dfb/C of 11f6fecb00005c6636e7ea541ff70dfb into c38cf285c47348c493ba151b8373d971(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T09:00:43,462 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 11f6fecb00005c6636e7ea541ff70dfb: 2024-11-20T09:00:43,462 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., storeName=11f6fecb00005c6636e7ea541ff70dfb/C, priority=13, startTime=1732093242996; duration=0sec 2024-11-20T09:00:43,462 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T09:00:43,462 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 11f6fecb00005c6636e7ea541ff70dfb:C 2024-11-20T09:00:43,573 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=264 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/A/b2d6779930e549af96d69528268996b3 2024-11-20T09:00:43,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/B/f647fc6a8f6146a2adbf52b2a5450542 is 50, key is test_row_0/B:col10/1732093241774/Put/seqid=0 2024-11-20T09:00:43,589 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742162_1338 (size=12301) 2024-11-20T09:00:43,589 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=264 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/B/f647fc6a8f6146a2adbf52b2a5450542 2024-11-20T09:00:43,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/C/7a1697d12aae47b7a4eef2c665800f1f is 50, key is test_row_0/C:col10/1732093241774/Put/seqid=0 2024-11-20T09:00:43,602 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742163_1339 (size=12301) 2024-11-20T09:00:43,602 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=264 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/C/7a1697d12aae47b7a4eef2c665800f1f 2024-11-20T09:00:43,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/A/b2d6779930e549af96d69528268996b3 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/A/b2d6779930e549af96d69528268996b3 2024-11-20T09:00:43,610 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/A/b2d6779930e549af96d69528268996b3, entries=150, sequenceid=264, filesize=12.0 K 2024-11-20T09:00:43,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/B/f647fc6a8f6146a2adbf52b2a5450542 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/B/f647fc6a8f6146a2adbf52b2a5450542 2024-11-20T09:00:43,615 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/B/f647fc6a8f6146a2adbf52b2a5450542, entries=150, sequenceid=264, filesize=12.0 K 2024-11-20T09:00:43,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/C/7a1697d12aae47b7a4eef2c665800f1f as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/C/7a1697d12aae47b7a4eef2c665800f1f 2024-11-20T09:00:43,620 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/C/7a1697d12aae47b7a4eef2c665800f1f, entries=150, sequenceid=264, filesize=12.0 K 2024-11-20T09:00:43,620 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=0 B/0 for 11f6fecb00005c6636e7ea541ff70dfb in 477ms, sequenceid=264, compaction requested=false 2024-11-20T09:00:43,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2603): Flush status journal for 11f6fecb00005c6636e7ea541ff70dfb: 2024-11-20T09:00:43,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb. 2024-11-20T09:00:43,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=73 2024-11-20T09:00:43,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.HMaster(4169): Remote procedure done, pid=73 2024-11-20T09:00:43,625 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=73, resume processing ppid=72 2024-11-20T09:00:43,625 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, ppid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5520 sec 2024-11-20T09:00:43,627 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees in 1.5580 sec 2024-11-20T09:00:43,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(8855): Flush requested on 11f6fecb00005c6636e7ea541ff70dfb 2024-11-20T09:00:43,940 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T09:00:43,940 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 11f6fecb00005c6636e7ea541ff70dfb 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-20T09:00:43,944 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/A/024c298818824097abc1d147b1cc1038 is 50, key is test_row_0/A:col10/1732093243937/Put/seqid=0 2024-11-20T09:00:43,953 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742164_1340 (size=14741) 2024-11-20T09:00:43,980 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:43,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58510 deadline: 1732093303976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:43,981 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:43,981 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:43,981 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:00:43,984 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:43,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58468 deadline: 1732093303979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:43,984 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:43,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58448 deadline: 1732093303980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:43,985 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:43,985 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:43,985 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:00:43,985 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:43,985 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:43,985 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:00:44,090 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:44,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58510 deadline: 1732093304087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:44,091 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:44,091 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:44,091 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:44,091 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:00:44,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58468 deadline: 1732093304087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:44,091 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:44,091 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:44,091 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:44,091 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:00:44,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58448 deadline: 1732093304087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:44,091 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:44,092 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:44,092 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:00:44,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-20T09:00:44,218 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T09:00:44,218 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T09:00:44,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] procedure2.ProcedureExecutor(1139): Stored pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees 2024-11-20T09:00:44,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-20T09:00:44,221 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T09:00:44,221 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T09:00:44,222 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=74, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T09:00:44,300 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:44,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58510 deadline: 1732093304297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:44,300 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:44,300 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:44,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58468 deadline: 1732093304297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:44,300 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:44,300 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:00:44,300 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:44,300 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:44,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58448 deadline: 1732093304297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:44,301 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:44,301 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:00:44,301 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:44,301 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:44,301 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:00:44,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-20T09:00:44,354 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=278 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/A/024c298818824097abc1d147b1cc1038 2024-11-20T09:00:44,362 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/B/8445dfb0fd5844dc8c27f8fe57383943 is 50, key is test_row_0/B:col10/1732093243937/Put/seqid=0 2024-11-20T09:00:44,365 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742165_1341 (size=12301) 2024-11-20T09:00:44,374 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-20T09:00:44,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb. 2024-11-20T09:00:44,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb. as already flushing 2024-11-20T09:00:44,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb. 2024-11-20T09:00:44,374 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 11f6fecb00005c6636e7ea541ff70dfb, NAME => 'TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:44,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 11f6fecb00005c6636e7ea541ff70dfb, NAME => 'TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:44,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 11f6fecb00005c6636e7ea541ff70dfb, NAME => 'TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 11f6fecb00005c6636e7ea541ff70dfb, NAME => 'TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:44,527 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-20T09:00:44,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb. 2024-11-20T09:00:44,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb. as already flushing 2024-11-20T09:00:44,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb. 2024-11-20T09:00:44,527 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 11f6fecb00005c6636e7ea541ff70dfb, NAME => 'TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:44,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 11f6fecb00005c6636e7ea541ff70dfb, NAME => 'TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:44,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 11f6fecb00005c6636e7ea541ff70dfb, NAME => 'TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 11f6fecb00005c6636e7ea541ff70dfb, NAME => 'TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:44,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-20T09:00:44,609 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:44,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58510 deadline: 1732093304607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:44,610 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:44,609 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:44,610 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:44,610 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:00:44,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58448 deadline: 1732093304607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:44,610 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:44,610 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:44,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58468 deadline: 1732093304607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:44,610 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:44,610 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:00:44,610 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:44,610 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:44,610 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:00:44,680 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-20T09:00:44,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb. 2024-11-20T09:00:44,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb. as already flushing 2024-11-20T09:00:44,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb. 2024-11-20T09:00:44,681 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 11f6fecb00005c6636e7ea541ff70dfb, NAME => 'TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:44,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 11f6fecb00005c6636e7ea541ff70dfb, NAME => 'TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:44,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 11f6fecb00005c6636e7ea541ff70dfb, NAME => 'TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 11f6fecb00005c6636e7ea541ff70dfb, NAME => 'TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:44,766 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=278 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/B/8445dfb0fd5844dc8c27f8fe57383943 2024-11-20T09:00:44,774 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/C/1584ebbccfa740d7add5768fe6f139a9 is 50, key is test_row_0/C:col10/1732093243937/Put/seqid=0 2024-11-20T09:00:44,781 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742166_1342 (size=12301) 2024-11-20T09:00:44,782 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=278 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/C/1584ebbccfa740d7add5768fe6f139a9 2024-11-20T09:00:44,786 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/A/024c298818824097abc1d147b1cc1038 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/A/024c298818824097abc1d147b1cc1038 2024-11-20T09:00:44,791 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/A/024c298818824097abc1d147b1cc1038, entries=200, sequenceid=278, filesize=14.4 K 2024-11-20T09:00:44,792 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/B/8445dfb0fd5844dc8c27f8fe57383943 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/B/8445dfb0fd5844dc8c27f8fe57383943 2024-11-20T09:00:44,795 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/B/8445dfb0fd5844dc8c27f8fe57383943, entries=150, sequenceid=278, filesize=12.0 K 2024-11-20T09:00:44,796 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/C/1584ebbccfa740d7add5768fe6f139a9 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/C/1584ebbccfa740d7add5768fe6f139a9 2024-11-20T09:00:44,800 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/C/1584ebbccfa740d7add5768fe6f139a9, entries=150, sequenceid=278, filesize=12.0 K 2024-11-20T09:00:44,801 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 11f6fecb00005c6636e7ea541ff70dfb in 861ms, sequenceid=278, compaction requested=true 2024-11-20T09:00:44,801 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 11f6fecb00005c6636e7ea541ff70dfb: 2024-11-20T09:00:44,801 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 11f6fecb00005c6636e7ea541ff70dfb:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T09:00:44,801 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T09:00:44,801 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T09:00:44,801 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 11f6fecb00005c6636e7ea541ff70dfb:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T09:00:44,801 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T09:00:44,801 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 11f6fecb00005c6636e7ea541ff70dfb:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T09:00:44,801 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T09:00:44,801 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T09:00:44,802 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39773 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T09:00:44,802 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HStore(1541): 11f6fecb00005c6636e7ea541ff70dfb/A is initiating minor compaction (all files) 2024-11-20T09:00:44,803 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 11f6fecb00005c6636e7ea541ff70dfb/A in TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb. 2024-11-20T09:00:44,803 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/A/1923e25e14774215b677c209bfc5b85e, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/A/b2d6779930e549af96d69528268996b3, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/A/024c298818824097abc1d147b1cc1038] into tmpdir=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp, totalSize=38.8 K 2024-11-20T09:00:44,803 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1923e25e14774215b677c209bfc5b85e, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=242, earliestPutTs=1732093241107 2024-11-20T09:00:44,803 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37333 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T09:00:44,804 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1541): 11f6fecb00005c6636e7ea541ff70dfb/B is initiating minor compaction (all files) 2024-11-20T09:00:44,804 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 11f6fecb00005c6636e7ea541ff70dfb/B in TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb. 2024-11-20T09:00:44,804 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/B/f97fc93675a141cabe97a8bb7d72cb10, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/B/f647fc6a8f6146a2adbf52b2a5450542, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/B/8445dfb0fd5844dc8c27f8fe57383943] into tmpdir=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp, totalSize=36.5 K 2024-11-20T09:00:44,804 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.Compactor(225): Compacting b2d6779930e549af96d69528268996b3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=264, earliestPutTs=1732093241768 2024-11-20T09:00:44,804 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting f97fc93675a141cabe97a8bb7d72cb10, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=242, earliestPutTs=1732093241107 2024-11-20T09:00:44,804 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.Compactor(225): Compacting 024c298818824097abc1d147b1cc1038, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1732093243935 2024-11-20T09:00:44,805 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting f647fc6a8f6146a2adbf52b2a5450542, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=264, earliestPutTs=1732093241768 2024-11-20T09:00:44,805 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting 8445dfb0fd5844dc8c27f8fe57383943, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1732093243937 2024-11-20T09:00:44,815 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 11f6fecb00005c6636e7ea541ff70dfb#A#compaction#291 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-20T09:00:44,816 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 11f6fecb00005c6636e7ea541ff70dfb#B#compaction#292 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T09:00:44,816 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/A/9c6ee43cfb2e46e697bdfdcdf42709c1 is 50, key is test_row_0/A:col10/1732093243937/Put/seqid=0 2024-11-20T09:00:44,816 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/B/126b6932c41a4686930eb400ecc3eb71 is 50, key is test_row_0/B:col10/1732093243937/Put/seqid=0 2024-11-20T09:00:44,830 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742167_1343 (size=12983) 2024-11-20T09:00:44,832 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742168_1344 (size=12983) 2024-11-20T09:00:44,833 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-20T09:00:44,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb. 2024-11-20T09:00:44,834 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2902): Flushing 11f6fecb00005c6636e7ea541ff70dfb 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-20T09:00:44,840 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/A/9c6ee43cfb2e46e697bdfdcdf42709c1 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/A/9c6ee43cfb2e46e697bdfdcdf42709c1 2024-11-20T09:00:44,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/A/1e822da94f664d80b56c0667f388e38a is 50, key is test_row_0/A:col10/1732093243978/Put/seqid=0 2024-11-20T09:00:44,846 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742169_1345 (size=12301) 2024-11-20T09:00:44,847 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 11f6fecb00005c6636e7ea541ff70dfb/A of 11f6fecb00005c6636e7ea541ff70dfb into 9c6ee43cfb2e46e697bdfdcdf42709c1(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T09:00:44,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-20T09:00:44,847 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 11f6fecb00005c6636e7ea541ff70dfb: 2024-11-20T09:00:44,847 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., storeName=11f6fecb00005c6636e7ea541ff70dfb/A, priority=13, startTime=1732093244801; duration=0sec 2024-11-20T09:00:44,847 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=303 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/A/1e822da94f664d80b56c0667f388e38a 2024-11-20T09:00:44,847 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T09:00:44,847 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 11f6fecb00005c6636e7ea541ff70dfb:A 2024-11-20T09:00:44,848 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T09:00:44,849 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37333 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T09:00:44,849 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HStore(1541): 11f6fecb00005c6636e7ea541ff70dfb/C is initiating minor compaction (all files) 2024-11-20T09:00:44,850 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 11f6fecb00005c6636e7ea541ff70dfb/C in TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb. 2024-11-20T09:00:44,850 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/C/c38cf285c47348c493ba151b8373d971, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/C/7a1697d12aae47b7a4eef2c665800f1f, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/C/1584ebbccfa740d7add5768fe6f139a9] into tmpdir=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp, totalSize=36.5 K 2024-11-20T09:00:44,850 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.Compactor(225): Compacting c38cf285c47348c493ba151b8373d971, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=242, earliestPutTs=1732093241107 2024-11-20T09:00:44,850 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7a1697d12aae47b7a4eef2c665800f1f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=264, earliestPutTs=1732093241768 2024-11-20T09:00:44,851 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1584ebbccfa740d7add5768fe6f139a9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1732093243937 2024-11-20T09:00:44,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/B/335ef6366f924d7aaa59cf5d394ebdc7 is 50, key is test_row_0/B:col10/1732093243978/Put/seqid=0 2024-11-20T09:00:44,859 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 11f6fecb00005c6636e7ea541ff70dfb#C#compaction#295 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T09:00:44,859 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/C/6cb47df77d87481283529e7145462664 is 50, key is test_row_0/C:col10/1732093243937/Put/seqid=0 2024-11-20T09:00:44,869 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742170_1346 (size=12301) 2024-11-20T09:00:44,875 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742171_1347 (size=12983) 2024-11-20T09:00:44,881 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/C/6cb47df77d87481283529e7145462664 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/C/6cb47df77d87481283529e7145462664 2024-11-20T09:00:44,890 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 11f6fecb00005c6636e7ea541ff70dfb/C of 11f6fecb00005c6636e7ea541ff70dfb into 6cb47df77d87481283529e7145462664(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T09:00:44,890 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 11f6fecb00005c6636e7ea541ff70dfb: 2024-11-20T09:00:44,890 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., storeName=11f6fecb00005c6636e7ea541ff70dfb/C, priority=13, startTime=1732093244801; duration=0sec 2024-11-20T09:00:44,890 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T09:00:44,890 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 11f6fecb00005c6636e7ea541ff70dfb:C 2024-11-20T09:00:45,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(8855): Flush requested on 11f6fecb00005c6636e7ea541ff70dfb 2024-11-20T09:00:45,120 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb. as already flushing 2024-11-20T09:00:45,132 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:45,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58510 deadline: 1732093305129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:45,133 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:45,133 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:45,133 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:00:45,134 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:45,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58448 deadline: 1732093305130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:45,135 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:45,135 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:45,135 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:45,135 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:00:45,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58468 deadline: 1732093305130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:45,135 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:45,135 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:45,135 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:00:45,237 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/B/126b6932c41a4686930eb400ecc3eb71 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/B/126b6932c41a4686930eb400ecc3eb71 2024-11-20T09:00:45,240 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:45,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58510 deadline: 1732093305236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:45,240 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:45,241 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:45,241 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:00:45,241 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 11f6fecb00005c6636e7ea541ff70dfb/B of 11f6fecb00005c6636e7ea541ff70dfb into 126b6932c41a4686930eb400ecc3eb71(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T09:00:45,241 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 11f6fecb00005c6636e7ea541ff70dfb: 2024-11-20T09:00:45,241 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., storeName=11f6fecb00005c6636e7ea541ff70dfb/B, priority=13, startTime=1732093244801; duration=0sec 2024-11-20T09:00:45,241 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T09:00:45,241 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 11f6fecb00005c6636e7ea541ff70dfb:B 2024-11-20T09:00:45,249 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:45,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58448 deadline: 1732093305247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:45,249 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:45,249 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:45,249 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:45,249 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:00:45,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58468 deadline: 1732093305247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:45,249 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:45,249 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:45,250 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:00:45,270 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=303 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/B/335ef6366f924d7aaa59cf5d394ebdc7 2024-11-20T09:00:45,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/C/a106995fc55f44489ae2b67fe8ab49d7 is 50, key is test_row_0/C:col10/1732093243978/Put/seqid=0 2024-11-20T09:00:45,280 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742172_1348 (size=12301) 2024-11-20T09:00:45,281 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=303 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/C/a106995fc55f44489ae2b67fe8ab49d7 2024-11-20T09:00:45,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/A/1e822da94f664d80b56c0667f388e38a as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/A/1e822da94f664d80b56c0667f388e38a 2024-11-20T09:00:45,288 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/A/1e822da94f664d80b56c0667f388e38a, entries=150, sequenceid=303, filesize=12.0 K 2024-11-20T09:00:45,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/B/335ef6366f924d7aaa59cf5d394ebdc7 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/B/335ef6366f924d7aaa59cf5d394ebdc7 2024-11-20T09:00:45,292 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/B/335ef6366f924d7aaa59cf5d394ebdc7, entries=150, sequenceid=303, filesize=12.0 K 2024-11-20T09:00:45,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/C/a106995fc55f44489ae2b67fe8ab49d7 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/C/a106995fc55f44489ae2b67fe8ab49d7 2024-11-20T09:00:45,297 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/C/a106995fc55f44489ae2b67fe8ab49d7, entries=150, sequenceid=303, filesize=12.0 K 2024-11-20T09:00:45,298 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=60.38 KB/61830 for 11f6fecb00005c6636e7ea541ff70dfb in 464ms, sequenceid=303, compaction requested=false 2024-11-20T09:00:45,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2603): Flush status journal for 11f6fecb00005c6636e7ea541ff70dfb: 2024-11-20T09:00:45,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb. 2024-11-20T09:00:45,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=75 2024-11-20T09:00:45,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.HMaster(4169): Remote procedure done, pid=75 2024-11-20T09:00:45,301 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=75, resume processing ppid=74 2024-11-20T09:00:45,301 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, ppid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0780 sec 2024-11-20T09:00:45,303 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees in 1.0840 sec 2024-11-20T09:00:45,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-20T09:00:45,357 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T09:00:45,357 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T09:00:45,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] procedure2.ProcedureExecutor(1139): Stored pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees 2024-11-20T09:00:45,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-20T09:00:45,359 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T09:00:45,360 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T09:00:45,360 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T09:00:45,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(8855): Flush requested on 11f6fecb00005c6636e7ea541ff70dfb 2024-11-20T09:00:45,449 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T09:00:45,449 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 11f6fecb00005c6636e7ea541ff70dfb 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-20T09:00:45,454 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/A/67f870e81e784985b46cfb1c00573123 is 50, key is test_row_0/A:col10/1732093245129/Put/seqid=0 2024-11-20T09:00:45,458 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742173_1349 (size=14741) 2024-11-20T09:00:45,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-20T09:00:45,518 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-20T09:00:45,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb. 2024-11-20T09:00:45,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb. as already flushing 2024-11-20T09:00:45,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb. 2024-11-20T09:00:45,518 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 11f6fecb00005c6636e7ea541ff70dfb, NAME => 'TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:45,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 11f6fecb00005c6636e7ea541ff70dfb, NAME => 'TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:45,519 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:45,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58510 deadline: 1732093305513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:45,519 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:45,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58448 deadline: 1732093305513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:45,519 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:45,519 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:45,519 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:00:45,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 11f6fecb00005c6636e7ea541ff70dfb, NAME => 'TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 11f6fecb00005c6636e7ea541ff70dfb, NAME => 'TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:45,519 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:45,519 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:45,519 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:45,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58468 deadline: 1732093305514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:45,520 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:00:45,520 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:45,520 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:45,520 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:00:45,629 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:45,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58510 deadline: 1732093305626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:45,629 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:45,629 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:45,629 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:45,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58448 deadline: 1732093305626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:45,629 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:00:45,629 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:45,630 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:45,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58468 deadline: 1732093305626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:45,630 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:45,630 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:00:45,630 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:45,630 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:45,630 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:00:45,671 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-20T09:00:45,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb. 2024-11-20T09:00:45,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb. as already flushing 2024-11-20T09:00:45,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb. 2024-11-20T09:00:45,672 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 11f6fecb00005c6636e7ea541ff70dfb, NAME => 'TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:45,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 11f6fecb00005c6636e7ea541ff70dfb, NAME => 'TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:45,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 11f6fecb00005c6636e7ea541ff70dfb, NAME => 'TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 11f6fecb00005c6636e7ea541ff70dfb, NAME => 'TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:45,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-20T09:00:45,824 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-20T09:00:45,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb. 2024-11-20T09:00:45,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb. as already flushing 2024-11-20T09:00:45,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb. 2024-11-20T09:00:45,825 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 11f6fecb00005c6636e7ea541ff70dfb, NAME => 'TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:45,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 11f6fecb00005c6636e7ea541ff70dfb, NAME => 'TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:45,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 11f6fecb00005c6636e7ea541ff70dfb, NAME => 'TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 11f6fecb00005c6636e7ea541ff70dfb, NAME => 'TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:45,841 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:45,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58510 deadline: 1732093305837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:45,841 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:45,842 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:45,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58468 deadline: 1732093305837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:45,842 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:45,842 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:00:45,842 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:45,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58448 deadline: 1732093305838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:45,842 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:45,842 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:45,842 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:00:45,842 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:45,842 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:45,842 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:00:45,859 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=319 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/A/67f870e81e784985b46cfb1c00573123 2024-11-20T09:00:45,866 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/B/cfc15da4f79a4f7e896a32b1b58b42cf is 50, key is test_row_0/B:col10/1732093245129/Put/seqid=0 2024-11-20T09:00:45,881 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742174_1350 (size=12301) 2024-11-20T09:00:45,977 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-20T09:00:45,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb. 2024-11-20T09:00:45,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb. as already flushing 2024-11-20T09:00:45,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb. 2024-11-20T09:00:45,978 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 11f6fecb00005c6636e7ea541ff70dfb, NAME => 'TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:45,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 11f6fecb00005c6636e7ea541ff70dfb, NAME => 'TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:45,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 11f6fecb00005c6636e7ea541ff70dfb, NAME => 'TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 11f6fecb00005c6636e7ea541ff70dfb, NAME => 'TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:45,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-20T09:00:46,131 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-20T09:00:46,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb. 2024-11-20T09:00:46,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb. as already flushing 2024-11-20T09:00:46,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb. 2024-11-20T09:00:46,131 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 11f6fecb00005c6636e7ea541ff70dfb, NAME => 'TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:46,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 11f6fecb00005c6636e7ea541ff70dfb, NAME => 'TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:46,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 11f6fecb00005c6636e7ea541ff70dfb, NAME => 'TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 11f6fecb00005c6636e7ea541ff70dfb, NAME => 'TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:46,150 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:46,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58510 deadline: 1732093306147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:46,150 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:46,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58468 deadline: 1732093306147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:46,150 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:46,150 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:46,150 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:00:46,150 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:46,150 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:46,150 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:46,150 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:00:46,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58448 deadline: 1732093306147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:46,151 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:46,151 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:46,151 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:00:46,281 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=319 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/B/cfc15da4f79a4f7e896a32b1b58b42cf 2024-11-20T09:00:46,284 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-20T09:00:46,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb. 2024-11-20T09:00:46,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb. as already flushing 2024-11-20T09:00:46,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb. 2024-11-20T09:00:46,285 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 11f6fecb00005c6636e7ea541ff70dfb, NAME => 'TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:46,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 11f6fecb00005c6636e7ea541ff70dfb, NAME => 'TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:46,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 11f6fecb00005c6636e7ea541ff70dfb, NAME => 'TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 11f6fecb00005c6636e7ea541ff70dfb, NAME => 'TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:46,288 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/C/b9b9ac335bbd4fa09be2f104acfd1fcc is 50, key is test_row_0/C:col10/1732093245129/Put/seqid=0 2024-11-20T09:00:46,292 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742175_1351 (size=12301) 2024-11-20T09:00:46,438 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-20T09:00:46,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb. 2024-11-20T09:00:46,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb. as already flushing 2024-11-20T09:00:46,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb. 2024-11-20T09:00:46,438 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 11f6fecb00005c6636e7ea541ff70dfb, NAME => 'TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:46,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 11f6fecb00005c6636e7ea541ff70dfb, NAME => 'TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:46,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 11f6fecb00005c6636e7ea541ff70dfb, NAME => 'TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 11f6fecb00005c6636e7ea541ff70dfb, NAME => 'TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:46,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-20T09:00:46,591 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-20T09:00:46,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb. 2024-11-20T09:00:46,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb. as already flushing 2024-11-20T09:00:46,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb. 2024-11-20T09:00:46,592 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 11f6fecb00005c6636e7ea541ff70dfb, NAME => 'TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:46,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 11f6fecb00005c6636e7ea541ff70dfb, NAME => 'TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:46,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 11f6fecb00005c6636e7ea541ff70dfb, NAME => 'TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 11f6fecb00005c6636e7ea541ff70dfb, NAME => 'TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:46,660 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:46,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58510 deadline: 1732093306657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:46,660 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:46,660 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:46,660 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:46,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58468 deadline: 1732093306657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:46,660 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:00:46,661 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:46,661 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:46,661 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:46,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58448 deadline: 1732093306657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:46,661 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:00:46,661 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:46,661 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:46,661 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:00:46,693 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=319 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/C/b9b9ac335bbd4fa09be2f104acfd1fcc 2024-11-20T09:00:46,697 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/A/67f870e81e784985b46cfb1c00573123 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/A/67f870e81e784985b46cfb1c00573123 2024-11-20T09:00:46,700 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/A/67f870e81e784985b46cfb1c00573123, entries=200, sequenceid=319, filesize=14.4 K 2024-11-20T09:00:46,701 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/B/cfc15da4f79a4f7e896a32b1b58b42cf as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/B/cfc15da4f79a4f7e896a32b1b58b42cf 2024-11-20T09:00:46,705 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/B/cfc15da4f79a4f7e896a32b1b58b42cf, entries=150, sequenceid=319, filesize=12.0 K 2024-11-20T09:00:46,706 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/C/b9b9ac335bbd4fa09be2f104acfd1fcc as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/C/b9b9ac335bbd4fa09be2f104acfd1fcc 2024-11-20T09:00:46,710 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/C/b9b9ac335bbd4fa09be2f104acfd1fcc, entries=150, sequenceid=319, filesize=12.0 K 2024-11-20T09:00:46,710 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 11f6fecb00005c6636e7ea541ff70dfb in 1261ms, sequenceid=319, compaction requested=true 2024-11-20T09:00:46,711 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 11f6fecb00005c6636e7ea541ff70dfb: 2024-11-20T09:00:46,711 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 11f6fecb00005c6636e7ea541ff70dfb:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T09:00:46,711 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T09:00:46,711 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T09:00:46,711 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 11f6fecb00005c6636e7ea541ff70dfb:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T09:00:46,711 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T09:00:46,711 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 11f6fecb00005c6636e7ea541ff70dfb:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T09:00:46,711 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T09:00:46,711 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T09:00:46,712 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40025 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T09:00:46,712 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HStore(1541): 11f6fecb00005c6636e7ea541ff70dfb/A is initiating minor compaction (all files) 2024-11-20T09:00:46,712 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 11f6fecb00005c6636e7ea541ff70dfb/A in TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb. 2024-11-20T09:00:46,712 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/A/9c6ee43cfb2e46e697bdfdcdf42709c1, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/A/1e822da94f664d80b56c0667f388e38a, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/A/67f870e81e784985b46cfb1c00573123] into tmpdir=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp, totalSize=39.1 K 2024-11-20T09:00:46,712 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T09:00:46,713 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1541): 11f6fecb00005c6636e7ea541ff70dfb/B is initiating minor compaction (all files) 2024-11-20T09:00:46,713 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 11f6fecb00005c6636e7ea541ff70dfb/B in TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb. 2024-11-20T09:00:46,713 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/B/126b6932c41a4686930eb400ecc3eb71, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/B/335ef6366f924d7aaa59cf5d394ebdc7, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/B/cfc15da4f79a4f7e896a32b1b58b42cf] into tmpdir=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp, totalSize=36.7 K 2024-11-20T09:00:46,713 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9c6ee43cfb2e46e697bdfdcdf42709c1, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1732093243937 2024-11-20T09:00:46,713 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting 126b6932c41a4686930eb400ecc3eb71, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1732093243937 2024-11-20T09:00:46,713 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1e822da94f664d80b56c0667f388e38a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=303, earliestPutTs=1732093243968 2024-11-20T09:00:46,713 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting 335ef6366f924d7aaa59cf5d394ebdc7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=303, earliestPutTs=1732093243968 2024-11-20T09:00:46,714 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting cfc15da4f79a4f7e896a32b1b58b42cf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=319, earliestPutTs=1732093245123 2024-11-20T09:00:46,714 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.Compactor(225): Compacting 67f870e81e784985b46cfb1c00573123, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=319, earliestPutTs=1732093245123 2024-11-20T09:00:46,721 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 11f6fecb00005c6636e7ea541ff70dfb#B#compaction#300 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-20T09:00:46,721 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/B/5282e2d05ada4e18a5d053ffc37f5e30 is 50, key is test_row_0/B:col10/1732093245129/Put/seqid=0 2024-11-20T09:00:46,721 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 11f6fecb00005c6636e7ea541ff70dfb#A#compaction#301 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T09:00:46,722 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/A/aace4f2251ad4faab88ec64b5cb0ec6f is 50, key is test_row_0/A:col10/1732093245129/Put/seqid=0 2024-11-20T09:00:46,735 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742177_1353 (size=13085) 2024-11-20T09:00:46,741 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/A/aace4f2251ad4faab88ec64b5cb0ec6f as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/A/aace4f2251ad4faab88ec64b5cb0ec6f 2024-11-20T09:00:46,745 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-20T09:00:46,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb. 2024-11-20T09:00:46,745 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2902): Flushing 11f6fecb00005c6636e7ea541ff70dfb 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-20T09:00:46,746 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 11f6fecb00005c6636e7ea541ff70dfb/A of 11f6fecb00005c6636e7ea541ff70dfb into aace4f2251ad4faab88ec64b5cb0ec6f(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T09:00:46,746 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 11f6fecb00005c6636e7ea541ff70dfb: 2024-11-20T09:00:46,746 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., storeName=11f6fecb00005c6636e7ea541ff70dfb/A, priority=13, startTime=1732093246711; duration=0sec 2024-11-20T09:00:46,746 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T09:00:46,746 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 11f6fecb00005c6636e7ea541ff70dfb:A 2024-11-20T09:00:46,746 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T09:00:46,748 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T09:00:46,748 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HStore(1541): 11f6fecb00005c6636e7ea541ff70dfb/C is initiating minor compaction (all files) 2024-11-20T09:00:46,748 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 11f6fecb00005c6636e7ea541ff70dfb/C in TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb. 2024-11-20T09:00:46,748 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/C/6cb47df77d87481283529e7145462664, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/C/a106995fc55f44489ae2b67fe8ab49d7, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/C/b9b9ac335bbd4fa09be2f104acfd1fcc] into tmpdir=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp, totalSize=36.7 K 2024-11-20T09:00:46,749 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6cb47df77d87481283529e7145462664, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1732093243937 2024-11-20T09:00:46,749 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.Compactor(225): Compacting a106995fc55f44489ae2b67fe8ab49d7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=303, earliestPutTs=1732093243968 2024-11-20T09:00:46,749 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.Compactor(225): Compacting b9b9ac335bbd4fa09be2f104acfd1fcc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=319, earliestPutTs=1732093245123 2024-11-20T09:00:46,749 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742176_1352 (size=13085) 2024-11-20T09:00:46,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/A/34901012e07c4467979474a7c63f5f96 is 50, key is test_row_0/A:col10/1732093245513/Put/seqid=0 2024-11-20T09:00:46,763 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 11f6fecb00005c6636e7ea541ff70dfb#C#compaction#303 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T09:00:46,764 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/C/524eda594786446aa31478f1bfa1fdc9 is 50, key is test_row_0/C:col10/1732093245129/Put/seqid=0 2024-11-20T09:00:46,779 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742179_1355 (size=13085) 2024-11-20T09:00:46,785 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/C/524eda594786446aa31478f1bfa1fdc9 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/C/524eda594786446aa31478f1bfa1fdc9 2024-11-20T09:00:46,786 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742178_1354 (size=12301) 2024-11-20T09:00:46,787 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=342 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/A/34901012e07c4467979474a7c63f5f96 2024-11-20T09:00:46,791 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 11f6fecb00005c6636e7ea541ff70dfb/C of 11f6fecb00005c6636e7ea541ff70dfb into 524eda594786446aa31478f1bfa1fdc9(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T09:00:46,791 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 11f6fecb00005c6636e7ea541ff70dfb: 2024-11-20T09:00:46,791 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., storeName=11f6fecb00005c6636e7ea541ff70dfb/C, priority=13, startTime=1732093246711; duration=0sec 2024-11-20T09:00:46,792 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T09:00:46,792 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 11f6fecb00005c6636e7ea541ff70dfb:C 2024-11-20T09:00:46,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/B/40ceb6a99520419cbdcd82b34ef9a008 is 50, key is test_row_0/B:col10/1732093245513/Put/seqid=0 2024-11-20T09:00:46,801 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742180_1356 (size=12301) 2024-11-20T09:00:47,157 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/B/5282e2d05ada4e18a5d053ffc37f5e30 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/B/5282e2d05ada4e18a5d053ffc37f5e30 2024-11-20T09:00:47,162 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 11f6fecb00005c6636e7ea541ff70dfb/B of 11f6fecb00005c6636e7ea541ff70dfb into 5282e2d05ada4e18a5d053ffc37f5e30(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T09:00:47,162 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 11f6fecb00005c6636e7ea541ff70dfb: 2024-11-20T09:00:47,162 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., storeName=11f6fecb00005c6636e7ea541ff70dfb/B, priority=13, startTime=1732093246711; duration=0sec 2024-11-20T09:00:47,162 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T09:00:47,162 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 11f6fecb00005c6636e7ea541ff70dfb:B 2024-11-20T09:00:47,202 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=342 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/B/40ceb6a99520419cbdcd82b34ef9a008 2024-11-20T09:00:47,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/C/b66527f898c74de8b1c64fcd0befa3cf is 50, key is test_row_0/C:col10/1732093245513/Put/seqid=0 2024-11-20T09:00:47,223 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742181_1357 (size=12301) 2024-11-20T09:00:47,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-20T09:00:47,624 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=342 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/C/b66527f898c74de8b1c64fcd0befa3cf 2024-11-20T09:00:47,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/A/34901012e07c4467979474a7c63f5f96 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/A/34901012e07c4467979474a7c63f5f96 2024-11-20T09:00:47,635 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/A/34901012e07c4467979474a7c63f5f96, entries=150, sequenceid=342, filesize=12.0 K 2024-11-20T09:00:47,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/B/40ceb6a99520419cbdcd82b34ef9a008 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/B/40ceb6a99520419cbdcd82b34ef9a008 2024-11-20T09:00:47,641 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/B/40ceb6a99520419cbdcd82b34ef9a008, entries=150, sequenceid=342, filesize=12.0 K 2024-11-20T09:00:47,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/C/b66527f898c74de8b1c64fcd0befa3cf as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/C/b66527f898c74de8b1c64fcd0befa3cf 2024-11-20T09:00:47,649 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/C/b66527f898c74de8b1c64fcd0befa3cf, entries=150, sequenceid=342, filesize=12.0 K 2024-11-20T09:00:47,650 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=0 B/0 for 11f6fecb00005c6636e7ea541ff70dfb in 905ms, sequenceid=342, compaction requested=false 2024-11-20T09:00:47,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2603): Flush status journal for 11f6fecb00005c6636e7ea541ff70dfb: 2024-11-20T09:00:47,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb. 2024-11-20T09:00:47,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=77 2024-11-20T09:00:47,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.HMaster(4169): Remote procedure done, pid=77 2024-11-20T09:00:47,654 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=77, resume processing ppid=76 2024-11-20T09:00:47,654 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, ppid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2920 sec 2024-11-20T09:00:47,656 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees in 2.2970 sec 2024-11-20T09:00:47,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(8855): Flush requested on 11f6fecb00005c6636e7ea541ff70dfb 2024-11-20T09:00:47,676 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T09:00:47,676 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 11f6fecb00005c6636e7ea541ff70dfb 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-20T09:00:47,681 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/A/c9fc2e03580a4e238c485eb9a465bb6b is 50, key is test_row_0/A:col10/1732093247675/Put/seqid=0 2024-11-20T09:00:47,687 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742182_1358 (size=12297) 2024-11-20T09:00:47,688 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=356 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/A/c9fc2e03580a4e238c485eb9a465bb6b 2024-11-20T09:00:47,694 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/B/8a34e27ac2354128a23e381666fb6a6f is 50, key is test_row_0/B:col10/1732093247675/Put/seqid=0 2024-11-20T09:00:47,698 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742183_1359 (size=9857) 2024-11-20T09:00:47,713 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:47,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58448 deadline: 1732093307709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:47,713 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:47,713 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:47,713 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:00:47,715 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:47,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58510 deadline: 1732093307711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:47,715 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:47,715 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:47,716 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:00:47,717 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:47,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58468 deadline: 1732093307713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:47,717 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:47,717 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:47,717 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:00:47,819 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:47,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58448 deadline: 1732093307817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:47,820 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:47,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58510 deadline: 1732093307817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:47,820 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:47,820 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:47,820 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:00:47,820 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:47,820 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:47,820 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:00:47,830 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:47,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58468 deadline: 1732093307826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:47,830 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:47,830 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:47,830 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:00:48,030 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:48,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58448 deadline: 1732093308027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:48,031 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:48,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58510 deadline: 1732093308027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:48,031 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:48,031 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:48,031 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:00:48,031 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:48,031 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:48,031 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:00:48,039 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:48,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58468 deadline: 1732093308036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:48,040 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:48,040 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:48,040 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:00:48,099 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=356 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/B/8a34e27ac2354128a23e381666fb6a6f 2024-11-20T09:00:48,106 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/C/3ecf0ab1108f4dfba126366c348adbc7 is 50, key is test_row_0/C:col10/1732093247675/Put/seqid=0 2024-11-20T09:00:48,110 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742184_1360 (size=9857) 2024-11-20T09:00:48,193 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-20T09:00:48,339 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:48,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58448 deadline: 1732093308337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:48,339 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:48,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58510 deadline: 1732093308337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:48,339 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:48,340 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:48,340 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:00:48,340 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:48,340 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:48,340 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:00:48,349 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:48,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58468 deadline: 1732093308347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:48,349 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:48,349 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:48,349 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:00:48,510 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=356 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/C/3ecf0ab1108f4dfba126366c348adbc7 2024-11-20T09:00:48,515 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/A/c9fc2e03580a4e238c485eb9a465bb6b as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/A/c9fc2e03580a4e238c485eb9a465bb6b 2024-11-20T09:00:48,518 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/A/c9fc2e03580a4e238c485eb9a465bb6b, entries=150, sequenceid=356, filesize=12.0 K 2024-11-20T09:00:48,519 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/B/8a34e27ac2354128a23e381666fb6a6f as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/B/8a34e27ac2354128a23e381666fb6a6f 2024-11-20T09:00:48,523 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/B/8a34e27ac2354128a23e381666fb6a6f, entries=100, sequenceid=356, filesize=9.6 K 2024-11-20T09:00:48,523 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/C/3ecf0ab1108f4dfba126366c348adbc7 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/C/3ecf0ab1108f4dfba126366c348adbc7 2024-11-20T09:00:48,527 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/C/3ecf0ab1108f4dfba126366c348adbc7, entries=100, sequenceid=356, filesize=9.6 K 2024-11-20T09:00:48,528 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 11f6fecb00005c6636e7ea541ff70dfb in 852ms, sequenceid=356, compaction requested=true 2024-11-20T09:00:48,528 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 11f6fecb00005c6636e7ea541ff70dfb: 2024-11-20T09:00:48,528 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 11f6fecb00005c6636e7ea541ff70dfb:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T09:00:48,528 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T09:00:48,528 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T09:00:48,528 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 11f6fecb00005c6636e7ea541ff70dfb:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T09:00:48,528 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T09:00:48,528 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 11f6fecb00005c6636e7ea541ff70dfb:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T09:00:48,528 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T09:00:48,528 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T09:00:48,529 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37683 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T09:00:48,529 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HStore(1541): 11f6fecb00005c6636e7ea541ff70dfb/A is initiating minor compaction (all files) 2024-11-20T09:00:48,529 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 11f6fecb00005c6636e7ea541ff70dfb/A in TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb. 2024-11-20T09:00:48,529 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/A/aace4f2251ad4faab88ec64b5cb0ec6f, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/A/34901012e07c4467979474a7c63f5f96, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/A/c9fc2e03580a4e238c485eb9a465bb6b] into tmpdir=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp, totalSize=36.8 K 2024-11-20T09:00:48,530 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35243 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T09:00:48,530 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1541): 11f6fecb00005c6636e7ea541ff70dfb/B is initiating minor compaction (all files) 2024-11-20T09:00:48,530 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.Compactor(225): Compacting aace4f2251ad4faab88ec64b5cb0ec6f, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=319, earliestPutTs=1732093245123 2024-11-20T09:00:48,530 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 11f6fecb00005c6636e7ea541ff70dfb/B in TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb. 2024-11-20T09:00:48,530 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/B/5282e2d05ada4e18a5d053ffc37f5e30, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/B/40ceb6a99520419cbdcd82b34ef9a008, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/B/8a34e27ac2354128a23e381666fb6a6f] into tmpdir=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp, totalSize=34.4 K 2024-11-20T09:00:48,530 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.Compactor(225): Compacting 34901012e07c4467979474a7c63f5f96, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=342, earliestPutTs=1732093245502 2024-11-20T09:00:48,530 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting 5282e2d05ada4e18a5d053ffc37f5e30, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=319, earliestPutTs=1732093245123 2024-11-20T09:00:48,530 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.Compactor(225): Compacting c9fc2e03580a4e238c485eb9a465bb6b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=356, earliestPutTs=1732093247673 2024-11-20T09:00:48,531 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting 40ceb6a99520419cbdcd82b34ef9a008, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=342, earliestPutTs=1732093245502 2024-11-20T09:00:48,531 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting 8a34e27ac2354128a23e381666fb6a6f, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=356, earliestPutTs=1732093247673 2024-11-20T09:00:48,538 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 11f6fecb00005c6636e7ea541ff70dfb#B#compaction#310 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-20T09:00:48,538 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 11f6fecb00005c6636e7ea541ff70dfb#A#compaction#309 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T09:00:48,538 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/A/6a7608898d4748d4b8992f173271816a is 50, key is test_row_0/A:col10/1732093247675/Put/seqid=0 2024-11-20T09:00:48,538 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/B/cac11e6925994a06a04895620fe116a8 is 50, key is test_row_0/B:col10/1732093247675/Put/seqid=0 2024-11-20T09:00:48,542 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742186_1362 (size=13187) 2024-11-20T09:00:48,543 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742185_1361 (size=13187) 2024-11-20T09:00:48,547 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/A/6a7608898d4748d4b8992f173271816a as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/A/6a7608898d4748d4b8992f173271816a 2024-11-20T09:00:48,548 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/B/cac11e6925994a06a04895620fe116a8 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/B/cac11e6925994a06a04895620fe116a8 2024-11-20T09:00:48,557 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 11f6fecb00005c6636e7ea541ff70dfb/A of 11f6fecb00005c6636e7ea541ff70dfb into 6a7608898d4748d4b8992f173271816a(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T09:00:48,557 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 11f6fecb00005c6636e7ea541ff70dfb: 2024-11-20T09:00:48,557 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., storeName=11f6fecb00005c6636e7ea541ff70dfb/A, priority=13, startTime=1732093248528; duration=0sec 2024-11-20T09:00:48,557 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T09:00:48,557 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 11f6fecb00005c6636e7ea541ff70dfb:A 2024-11-20T09:00:48,557 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T09:00:48,558 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 11f6fecb00005c6636e7ea541ff70dfb/B of 11f6fecb00005c6636e7ea541ff70dfb into cac11e6925994a06a04895620fe116a8(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T09:00:48,558 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 11f6fecb00005c6636e7ea541ff70dfb: 2024-11-20T09:00:48,558 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., storeName=11f6fecb00005c6636e7ea541ff70dfb/B, priority=13, startTime=1732093248528; duration=0sec 2024-11-20T09:00:48,558 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T09:00:48,558 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 11f6fecb00005c6636e7ea541ff70dfb:B 2024-11-20T09:00:48,558 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35243 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T09:00:48,559 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HStore(1541): 11f6fecb00005c6636e7ea541ff70dfb/C is initiating minor compaction (all files) 2024-11-20T09:00:48,559 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 11f6fecb00005c6636e7ea541ff70dfb/C in TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb. 2024-11-20T09:00:48,559 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/C/524eda594786446aa31478f1bfa1fdc9, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/C/b66527f898c74de8b1c64fcd0befa3cf, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/C/3ecf0ab1108f4dfba126366c348adbc7] into tmpdir=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp, totalSize=34.4 K 2024-11-20T09:00:48,559 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.Compactor(225): Compacting 524eda594786446aa31478f1bfa1fdc9, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=319, earliestPutTs=1732093245123 2024-11-20T09:00:48,559 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.Compactor(225): Compacting b66527f898c74de8b1c64fcd0befa3cf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=342, earliestPutTs=1732093245502 2024-11-20T09:00:48,560 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3ecf0ab1108f4dfba126366c348adbc7, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=356, earliestPutTs=1732093247673 2024-11-20T09:00:48,566 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 11f6fecb00005c6636e7ea541ff70dfb#C#compaction#311 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T09:00:48,566 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/C/b9a10ef5c89245b7be4a6c787210f3f9 is 50, key is test_row_0/C:col10/1732093247675/Put/seqid=0 2024-11-20T09:00:48,570 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742187_1363 (size=13187) 2024-11-20T09:00:48,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(8855): Flush requested on 11f6fecb00005c6636e7ea541ff70dfb 2024-11-20T09:00:48,850 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T09:00:48,850 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 11f6fecb00005c6636e7ea541ff70dfb 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-20T09:00:48,855 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/A/1ae95256b2ad45329c92eede78d6e983 is 50, key is test_row_0/A:col10/1732093247712/Put/seqid=0 2024-11-20T09:00:48,864 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742188_1364 (size=17181) 2024-11-20T09:00:48,893 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:48,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58510 deadline: 1732093308890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:48,894 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:48,894 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:48,894 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:00:48,894 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:48,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58468 deadline: 1732093308891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:48,895 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:48,895 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:48,895 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:00:48,895 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:48,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58448 deadline: 1732093308891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:48,895 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:48,895 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:48,895 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:00:48,976 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/C/b9a10ef5c89245b7be4a6c787210f3f9 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/C/b9a10ef5c89245b7be4a6c787210f3f9 2024-11-20T09:00:48,981 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 11f6fecb00005c6636e7ea541ff70dfb/C of 11f6fecb00005c6636e7ea541ff70dfb into b9a10ef5c89245b7be4a6c787210f3f9(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T09:00:48,981 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 11f6fecb00005c6636e7ea541ff70dfb: 2024-11-20T09:00:48,981 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., storeName=11f6fecb00005c6636e7ea541ff70dfb/C, priority=13, startTime=1732093248528; duration=0sec 2024-11-20T09:00:48,981 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T09:00:48,981 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 11f6fecb00005c6636e7ea541ff70dfb:C 2024-11-20T09:00:49,000 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:49,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58448 deadline: 1732093308997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:49,000 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:49,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58510 deadline: 1732093308997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:49,000 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:49,000 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:49,000 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:49,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58468 deadline: 1732093308997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:49,000 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:00:49,001 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:49,001 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:49,001 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:00:49,001 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:49,001 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:49,001 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:00:49,209 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:49,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58448 deadline: 1732093309207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:49,209 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:49,209 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:49,209 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:49,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58510 deadline: 1732093309207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:49,209 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:00:49,210 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:49,210 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:49,210 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:49,210 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:00:49,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58468 deadline: 1732093309207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:49,210 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:49,210 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:49,210 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:00:49,265 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=385 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/A/1ae95256b2ad45329c92eede78d6e983 2024-11-20T09:00:49,273 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/B/09b4a71772ed4dc0a0bb7bf0bfba96c0 is 50, key is test_row_0/B:col10/1732093247712/Put/seqid=0 2024-11-20T09:00:49,276 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742189_1365 (size=12301) 2024-11-20T09:00:49,518 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:49,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58448 deadline: 1732093309517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:49,519 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:49,519 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:49,519 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:49,519 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:00:49,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58510 deadline: 1732093309517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:49,519 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:49,519 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:49,519 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:00:49,519 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:49,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58468 deadline: 1732093309518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:49,519 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:49,519 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:49,519 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:00:49,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-20T09:00:49,527 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T09:00:49,528 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T09:00:49,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] procedure2.ProcedureExecutor(1139): Stored pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees 2024-11-20T09:00:49,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-20T09:00:49,530 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T09:00:49,530 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T09:00:49,531 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=78, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T09:00:49,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-20T09:00:49,677 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=385 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/B/09b4a71772ed4dc0a0bb7bf0bfba96c0 2024-11-20T09:00:49,683 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-20T09:00:49,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb. 2024-11-20T09:00:49,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb. as already flushing 2024-11-20T09:00:49,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb. 2024-11-20T09:00:49,683 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 11f6fecb00005c6636e7ea541ff70dfb, NAME => 'TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:49,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 11f6fecb00005c6636e7ea541ff70dfb, NAME => 'TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:49,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 11f6fecb00005c6636e7ea541ff70dfb, NAME => 'TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 11f6fecb00005c6636e7ea541ff70dfb, NAME => 'TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:49,690 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/C/fecb2746074648678b0f8a2336fa69fe is 50, key is test_row_0/C:col10/1732093247712/Put/seqid=0 2024-11-20T09:00:49,694 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742190_1366 (size=12301) 2024-11-20T09:00:49,836 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-20T09:00:49,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb. 2024-11-20T09:00:49,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb. as already flushing 2024-11-20T09:00:49,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb. 2024-11-20T09:00:49,837 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 11f6fecb00005c6636e7ea541ff70dfb, NAME => 'TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:49,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 11f6fecb00005c6636e7ea541ff70dfb, NAME => 'TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:49,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 11f6fecb00005c6636e7ea541ff70dfb, NAME => 'TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 11f6fecb00005c6636e7ea541ff70dfb, NAME => 'TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:49,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-20T09:00:49,990 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-20T09:00:49,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb. 2024-11-20T09:00:49,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb. as already flushing 2024-11-20T09:00:49,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb. 2024-11-20T09:00:49,990 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 11f6fecb00005c6636e7ea541ff70dfb, NAME => 'TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:49,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 11f6fecb00005c6636e7ea541ff70dfb, NAME => 'TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:49,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 11f6fecb00005c6636e7ea541ff70dfb, NAME => 'TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 11f6fecb00005c6636e7ea541ff70dfb, NAME => 'TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:50,030 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:50,031 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:50,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58510 deadline: 1732093310027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:50,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58448 deadline: 1732093310027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:50,031 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:50,031 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:50,031 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:50,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58468 deadline: 1732093310027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:50,031 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:50,031 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:50,031 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:00:50,031 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:00:50,031 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:50,031 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:50,032 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:00:50,095 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=385 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/C/fecb2746074648678b0f8a2336fa69fe 2024-11-20T09:00:50,099 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/A/1ae95256b2ad45329c92eede78d6e983 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/A/1ae95256b2ad45329c92eede78d6e983 2024-11-20T09:00:50,103 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/A/1ae95256b2ad45329c92eede78d6e983, entries=250, sequenceid=385, filesize=16.8 K 2024-11-20T09:00:50,104 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/B/09b4a71772ed4dc0a0bb7bf0bfba96c0 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/B/09b4a71772ed4dc0a0bb7bf0bfba96c0 2024-11-20T09:00:50,108 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/B/09b4a71772ed4dc0a0bb7bf0bfba96c0, entries=150, sequenceid=385, filesize=12.0 K 2024-11-20T09:00:50,108 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/C/fecb2746074648678b0f8a2336fa69fe as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/C/fecb2746074648678b0f8a2336fa69fe 2024-11-20T09:00:50,112 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/C/fecb2746074648678b0f8a2336fa69fe, entries=150, sequenceid=385, filesize=12.0 K 2024-11-20T09:00:50,113 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=46.96 KB/48090 for 11f6fecb00005c6636e7ea541ff70dfb in 1263ms, sequenceid=385, compaction requested=false 2024-11-20T09:00:50,113 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 11f6fecb00005c6636e7ea541ff70dfb: 2024-11-20T09:00:50,143 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-20T09:00:50,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb. 2024-11-20T09:00:50,143 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2902): Flushing 11f6fecb00005c6636e7ea541ff70dfb 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-20T09:00:50,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/A/49409cd49b8349b0b1b931753cc719c3 is 50, key is test_row_0/A:col10/1732093248889/Put/seqid=0 2024-11-20T09:00:50,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-20T09:00:50,159 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742191_1367 (size=12301) 2024-11-20T09:00:50,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(8855): Flush requested on 11f6fecb00005c6636e7ea541ff70dfb 2024-11-20T09:00:50,530 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb. as already flushing 2024-11-20T09:00:50,560 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=396 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/A/49409cd49b8349b0b1b931753cc719c3 2024-11-20T09:00:50,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/B/4dd0cc50724e4e11b7800db81dda6307 is 50, key is test_row_0/B:col10/1732093248889/Put/seqid=0 2024-11-20T09:00:50,571 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742192_1368 (size=12301) 2024-11-20T09:00:50,598 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:50,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58454 deadline: 1732093310594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:50,599 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:50,599 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:50,599 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:00:50,602 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:50,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58452 deadline: 1732093310597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:50,602 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:50,602 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:50,602 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:00:50,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-20T09:00:50,710 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:50,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58454 deadline: 1732093310707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:50,710 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:50,710 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:50,710 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:00:50,711 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:50,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58452 deadline: 1732093310707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:50,711 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:50,711 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:50,711 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:00:50,920 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:50,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58454 deadline: 1732093310917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:50,920 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:50,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58452 deadline: 1732093310917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:50,920 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:50,921 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:50,921 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:00:50,921 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:50,921 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:50,921 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:00:50,972 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=396 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/B/4dd0cc50724e4e11b7800db81dda6307 2024-11-20T09:00:50,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/C/523cc90a6c3b4dc3ae3e9f6eec68e5c3 is 50, key is test_row_0/C:col10/1732093248889/Put/seqid=0 2024-11-20T09:00:50,982 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742193_1369 (size=12301) 2024-11-20T09:00:50,993 INFO [Thread-1304 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1304. 2024-11-20T09:00:50,993 DEBUG [Thread-1304 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T09:00:50,993 DEBUG [Thread-1304 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T09:00:50,993 DEBUG [Thread-1304 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T09:00:50,993 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T09:00:50,994 INFO [Thread-1306 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1306. 2024-11-20T09:00:50,994 DEBUG [Thread-1306 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T09:00:50,994 DEBUG [Thread-1306 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T09:00:50,994 DEBUG [Thread-1306 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T09:00:50,994 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T09:00:50,996 INFO [Thread-1308 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1308. 2024-11-20T09:00:50,996 DEBUG [Thread-1308 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T09:00:50,996 DEBUG [Thread-1308 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T09:00:50,996 DEBUG [Thread-1308 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T09:00:50,996 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T09:00:50,997 INFO [Thread-1300 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1300. 2024-11-20T09:00:50,997 DEBUG [Thread-1300 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T09:00:50,997 DEBUG [Thread-1300 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T09:00:50,997 DEBUG [Thread-1300 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T09:00:50,997 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T09:00:50,998 INFO [Thread-1302 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1302. 2024-11-20T09:00:50,998 DEBUG [Thread-1302 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T09:00:50,998 DEBUG [Thread-1302 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T09:00:50,998 DEBUG [Thread-1302 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T09:00:50,998 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T09:00:51,037 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:51,037 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:51,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58510 deadline: 1732093311037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:51,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58468 deadline: 1732093311037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:51,038 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to be1eb2620e0e,34223,1732093159960 for 'test_row_0' in 11f6fecb00005c6636e7ea541ff70dfb of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2148 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-20T09:00:51,038 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to be1eb2620e0e,34223,1732093159960 for 'test_row_0' in 11f6fecb00005c6636e7ea541ff70dfb of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2147 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-20T09:00:51,038 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:51,038 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:51,038 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:51,038 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:51,039 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:00:51,039 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:00:51,047 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:51,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58448 deadline: 1732093311047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:51,047 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to be1eb2620e0e,34223,1732093159960 for 'test_row_1' in 11f6fecb00005c6636e7ea541ff70dfb of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2156 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-20T09:00:51,048 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:51,048 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:51,048 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:00:51,228 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:51,228 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:51,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58452 deadline: 1732093311227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:51,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58454 deadline: 1732093311227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:51,228 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:51,228 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:51,228 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:51,228 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:51,228 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:00:51,228 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:00:51,383 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=396 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/C/523cc90a6c3b4dc3ae3e9f6eec68e5c3 2024-11-20T09:00:51,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/A/49409cd49b8349b0b1b931753cc719c3 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/A/49409cd49b8349b0b1b931753cc719c3 2024-11-20T09:00:51,390 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/A/49409cd49b8349b0b1b931753cc719c3, entries=150, sequenceid=396, filesize=12.0 K 2024-11-20T09:00:51,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/B/4dd0cc50724e4e11b7800db81dda6307 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/B/4dd0cc50724e4e11b7800db81dda6307 2024-11-20T09:00:51,394 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/B/4dd0cc50724e4e11b7800db81dda6307, entries=150, sequenceid=396, filesize=12.0 K 2024-11-20T09:00:51,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/C/523cc90a6c3b4dc3ae3e9f6eec68e5c3 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/C/523cc90a6c3b4dc3ae3e9f6eec68e5c3 2024-11-20T09:00:51,398 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/C/523cc90a6c3b4dc3ae3e9f6eec68e5c3, entries=150, sequenceid=396, filesize=12.0 K 2024-11-20T09:00:51,398 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=154.31 KB/158010 for 11f6fecb00005c6636e7ea541ff70dfb in 1255ms, sequenceid=396, compaction requested=true 2024-11-20T09:00:51,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2603): Flush status journal for 11f6fecb00005c6636e7ea541ff70dfb: 2024-11-20T09:00:51,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb. 2024-11-20T09:00:51,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=79 2024-11-20T09:00:51,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.HMaster(4169): Remote procedure done, pid=79 2024-11-20T09:00:51,401 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=79, resume processing ppid=78 2024-11-20T09:00:51,401 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, ppid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8700 sec 2024-11-20T09:00:51,403 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees in 1.8740 sec 2024-11-20T09:00:51,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-20T09:00:51,687 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T09:00:51,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(8855): Flush requested on 11f6fecb00005c6636e7ea541ff70dfb 2024-11-20T09:00:51,739 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T09:00:51,739 INFO [Thread-1291 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1291. 2024-11-20T09:00:51,739 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 11f6fecb00005c6636e7ea541ff70dfb 3/3 column families, dataSize=167.72 KB heapSize=440.20 KB 2024-11-20T09:00:51,739 INFO [Thread-1297 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1297. 2024-11-20T09:00:51,739 DEBUG [Thread-1291 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T09:00:51,739 DEBUG [Thread-1291 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T09:00:51,739 DEBUG [Thread-1297 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T09:00:51,739 DEBUG [Thread-1297 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T09:00:51,739 DEBUG [Thread-1297 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T09:00:51,739 DEBUG [Thread-1291 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T09:00:51,739 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T09:00:51,739 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T09:00:51,743 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/A/d80088a4392f44bf8749b15543f6b3f0 is 50, key is test_row_0/A:col10/1732093251737/Put/seqid=0 2024-11-20T09:00:51,746 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742194_1370 (size=12301) 2024-11-20T09:00:52,147 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=424 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/A/d80088a4392f44bf8749b15543f6b3f0 2024-11-20T09:00:52,153 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/B/7149235b10d54d84b3f73e3ecac84707 is 50, key is test_row_0/B:col10/1732093251737/Put/seqid=0 2024-11-20T09:00:52,156 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742195_1371 (size=12301) 2024-11-20T09:00:52,557 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=424 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/B/7149235b10d54d84b3f73e3ecac84707 2024-11-20T09:00:52,562 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/C/3a8e6cdb12054c2a9889257c61f91277 is 50, key is test_row_0/C:col10/1732093251737/Put/seqid=0 2024-11-20T09:00:52,566 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742196_1372 (size=12301) 2024-11-20T09:00:52,966 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=424 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/C/3a8e6cdb12054c2a9889257c61f91277 2024-11-20T09:00:52,970 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/A/d80088a4392f44bf8749b15543f6b3f0 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/A/d80088a4392f44bf8749b15543f6b3f0 2024-11-20T09:00:52,973 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/A/d80088a4392f44bf8749b15543f6b3f0, entries=150, sequenceid=424, filesize=12.0 K 2024-11-20T09:00:52,974 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/B/7149235b10d54d84b3f73e3ecac84707 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/B/7149235b10d54d84b3f73e3ecac84707 2024-11-20T09:00:52,976 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/B/7149235b10d54d84b3f73e3ecac84707, entries=150, sequenceid=424, filesize=12.0 K 2024-11-20T09:00:52,977 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/C/3a8e6cdb12054c2a9889257c61f91277 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/C/3a8e6cdb12054c2a9889257c61f91277 2024-11-20T09:00:52,979 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/C/3a8e6cdb12054c2a9889257c61f91277, entries=150, sequenceid=424, filesize=12.0 K 2024-11-20T09:00:52,980 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~167.72 KB/171750, heapSize ~440.16 KB/450720, currentSize=0 B/0 for 11f6fecb00005c6636e7ea541ff70dfb in 1241ms, sequenceid=424, compaction requested=true 2024-11-20T09:00:52,980 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 11f6fecb00005c6636e7ea541ff70dfb: 2024-11-20T09:00:52,980 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 11f6fecb00005c6636e7ea541ff70dfb:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T09:00:52,980 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T09:00:52,981 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 11f6fecb00005c6636e7ea541ff70dfb:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T09:00:52,981 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T09:00:52,981 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 11f6fecb00005c6636e7ea541ff70dfb:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T09:00:52,981 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T09:00:52,981 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T09:00:52,981 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T09:00:52,981 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50090 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T09:00:52,981 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 54970 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T09:00:52,982 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1541): 11f6fecb00005c6636e7ea541ff70dfb/B is initiating minor compaction (all files) 2024-11-20T09:00:52,982 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HStore(1541): 11f6fecb00005c6636e7ea541ff70dfb/A is initiating minor compaction (all files) 2024-11-20T09:00:52,982 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 11f6fecb00005c6636e7ea541ff70dfb/B in TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb. 2024-11-20T09:00:52,982 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 11f6fecb00005c6636e7ea541ff70dfb/A in TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb. 2024-11-20T09:00:52,982 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/B/cac11e6925994a06a04895620fe116a8, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/B/09b4a71772ed4dc0a0bb7bf0bfba96c0, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/B/4dd0cc50724e4e11b7800db81dda6307, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/B/7149235b10d54d84b3f73e3ecac84707] into tmpdir=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp, totalSize=48.9 K 2024-11-20T09:00:52,982 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/A/6a7608898d4748d4b8992f173271816a, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/A/1ae95256b2ad45329c92eede78d6e983, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/A/49409cd49b8349b0b1b931753cc719c3, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/A/d80088a4392f44bf8749b15543f6b3f0] into tmpdir=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp, totalSize=53.7 K 2024-11-20T09:00:52,982 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting cac11e6925994a06a04895620fe116a8, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=356, earliestPutTs=1732093245502 2024-11-20T09:00:52,982 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6a7608898d4748d4b8992f173271816a, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=356, earliestPutTs=1732093245502 2024-11-20T09:00:52,982 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting 09b4a71772ed4dc0a0bb7bf0bfba96c0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=385, earliestPutTs=1732093247708 2024-11-20T09:00:52,982 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1ae95256b2ad45329c92eede78d6e983, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=385, earliestPutTs=1732093247708 2024-11-20T09:00:52,982 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting 4dd0cc50724e4e11b7800db81dda6307, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=396, earliestPutTs=1732093248888 2024-11-20T09:00:52,982 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.Compactor(225): Compacting 49409cd49b8349b0b1b931753cc719c3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=396, earliestPutTs=1732093248888 2024-11-20T09:00:52,983 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting 7149235b10d54d84b3f73e3ecac84707, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=424, earliestPutTs=1732093250593 2024-11-20T09:00:52,983 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.Compactor(225): Compacting d80088a4392f44bf8749b15543f6b3f0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=424, earliestPutTs=1732093250593 2024-11-20T09:00:52,990 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 11f6fecb00005c6636e7ea541ff70dfb#B#compaction#321 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-20T09:00:52,990 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 11f6fecb00005c6636e7ea541ff70dfb#A#compaction#322 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T09:00:52,991 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/B/bbf8d66daece4f8faeeca715cfa86914 is 50, key is test_row_0/B:col10/1732093251737/Put/seqid=0 2024-11-20T09:00:52,991 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/A/f13c9520bb414850a8a4427d0212ccb6 is 50, key is test_row_0/A:col10/1732093251737/Put/seqid=0 2024-11-20T09:00:52,994 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742198_1374 (size=13323) 2024-11-20T09:00:52,998 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742197_1373 (size=13323) 2024-11-20T09:00:53,048 INFO [Thread-1295 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1295. 2024-11-20T09:00:53,048 DEBUG [Thread-1295 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T09:00:53,048 DEBUG [Thread-1295 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T09:00:53,048 DEBUG [Thread-1295 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T09:00:53,048 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T09:00:53,058 INFO [Thread-1289 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1289. 2024-11-20T09:00:53,058 INFO [Thread-1293 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1293. 2024-11-20T09:00:53,058 DEBUG [Thread-1293 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T09:00:53,058 DEBUG [Thread-1289 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T09:00:53,058 DEBUG [Thread-1293 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T09:00:53,058 DEBUG [Thread-1289 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T09:00:53,059 DEBUG [Thread-1293 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T09:00:53,059 DEBUG [Thread-1289 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T09:00:53,059 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T09:00:53,059 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T09:00:53,059 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-20T09:00:53,059 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 89 2024-11-20T09:00:53,059 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 16 2024-11-20T09:00:53,059 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 85 2024-11-20T09:00:53,059 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 121 2024-11-20T09:00:53,059 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 24 2024-11-20T09:00:53,059 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-20T09:00:53,059 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3603 2024-11-20T09:00:53,059 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3634 2024-11-20T09:00:53,059 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3533 2024-11-20T09:00:53,059 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3577 2024-11-20T09:00:53,059 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3624 2024-11-20T09:00:53,059 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-20T09:00:53,059 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-20T09:00:53,059 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testGetAtomicity(AcidGuaranteesTestBase.java:107) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-20T09:00:53,059 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T09:00:53,059 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T09:00:53,059 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T09:00:53,060 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-20T09:00:53,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] procedure2.ProcedureExecutor(1139): Stored pid=80, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-20T09:00:53,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-20T09:00:53,063 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732093253063"}]},"ts":"1732093253063"} 2024-11-20T09:00:53,064 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-20T09:00:53,064 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-20T09:00:53,065 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=80, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-20T09:00:53,066 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=11f6fecb00005c6636e7ea541ff70dfb, UNASSIGN}] 2024-11-20T09:00:53,067 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=11f6fecb00005c6636e7ea541ff70dfb, UNASSIGN 2024-11-20T09:00:53,068 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=82 updating hbase:meta row=11f6fecb00005c6636e7ea541ff70dfb, regionState=CLOSING, regionLocation=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:53,069 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=11f6fecb00005c6636e7ea541ff70dfb, UNASSIGN because future has completed 2024-11-20T09:00:53,069 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-20T09:00:53,069 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=83, ppid=82, state=RUNNABLE, hasLock=false; CloseRegionProcedure 11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960}] 2024-11-20T09:00:53,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-20T09:00:53,221 INFO [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] handler.UnassignRegionHandler(122): Close 11f6fecb00005c6636e7ea541ff70dfb 2024-11-20T09:00:53,221 DEBUG [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-20T09:00:53,221 DEBUG [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1722): Closing 11f6fecb00005c6636e7ea541ff70dfb, disabling compactions & flushes 2024-11-20T09:00:53,221 DEBUG [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1993): waiting for 2 compactions to complete for region TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb. 2024-11-20T09:00:53,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-20T09:00:53,398 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/A/f13c9520bb414850a8a4427d0212ccb6 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/A/f13c9520bb414850a8a4427d0212ccb6 2024-11-20T09:00:53,402 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/B/bbf8d66daece4f8faeeca715cfa86914 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/B/bbf8d66daece4f8faeeca715cfa86914 2024-11-20T09:00:53,403 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 11f6fecb00005c6636e7ea541ff70dfb/A of 11f6fecb00005c6636e7ea541ff70dfb into f13c9520bb414850a8a4427d0212ccb6(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T09:00:53,403 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 11f6fecb00005c6636e7ea541ff70dfb: 2024-11-20T09:00:53,403 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., storeName=11f6fecb00005c6636e7ea541ff70dfb/A, priority=12, startTime=1732093252980; duration=0sec 2024-11-20T09:00:53,403 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T09:00:53,403 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 11f6fecb00005c6636e7ea541ff70dfb:A 2024-11-20T09:00:53,403 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb. because compaction request was cancelled 2024-11-20T09:00:53,403 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 11f6fecb00005c6636e7ea541ff70dfb:C 2024-11-20T09:00:53,406 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 11f6fecb00005c6636e7ea541ff70dfb/B of 11f6fecb00005c6636e7ea541ff70dfb into bbf8d66daece4f8faeeca715cfa86914(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T09:00:53,406 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 11f6fecb00005c6636e7ea541ff70dfb: 2024-11-20T09:00:53,406 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., storeName=11f6fecb00005c6636e7ea541ff70dfb/B, priority=12, startTime=1732093252980; duration=0sec 2024-11-20T09:00:53,406 INFO [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb. 2024-11-20T09:00:53,406 DEBUG [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb. 2024-11-20T09:00:53,406 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T09:00:53,406 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 11f6fecb00005c6636e7ea541ff70dfb:B 2024-11-20T09:00:53,406 DEBUG [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb. after waiting 0 ms 2024-11-20T09:00:53,406 DEBUG [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb. 2024-11-20T09:00:53,406 INFO [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(2902): Flushing 11f6fecb00005c6636e7ea541ff70dfb 3/3 column families, dataSize=20.13 KB heapSize=53.48 KB 2024-11-20T09:00:53,409 DEBUG [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/A/cff9586e8f774ef6b59bff9ef3373bdd is 50, key is test_row_0/A:col10/1732093253057/Put/seqid=0 2024-11-20T09:00:53,412 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742199_1375 (size=9857) 2024-11-20T09:00:53,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-20T09:00:53,813 INFO [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=432 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/A/cff9586e8f774ef6b59bff9ef3373bdd 2024-11-20T09:00:53,819 DEBUG [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/B/9cfe360cd7bf4c2b80ca2ac9ac35bbbe is 50, key is test_row_0/B:col10/1732093253057/Put/seqid=0 2024-11-20T09:00:53,822 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742200_1376 (size=9857) 2024-11-20T09:00:54,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-20T09:00:54,223 INFO [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=432 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/B/9cfe360cd7bf4c2b80ca2ac9ac35bbbe 2024-11-20T09:00:54,229 DEBUG [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/C/2e27a144a4274299aab39b51b6f591fb is 50, key is test_row_0/C:col10/1732093253057/Put/seqid=0 2024-11-20T09:00:54,232 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742201_1377 (size=9857) 2024-11-20T09:00:54,633 INFO [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=432 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/C/2e27a144a4274299aab39b51b6f591fb 2024-11-20T09:00:54,637 DEBUG [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/A/cff9586e8f774ef6b59bff9ef3373bdd as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/A/cff9586e8f774ef6b59bff9ef3373bdd 2024-11-20T09:00:54,640 INFO [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/A/cff9586e8f774ef6b59bff9ef3373bdd, entries=100, sequenceid=432, filesize=9.6 K 2024-11-20T09:00:54,640 DEBUG [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/B/9cfe360cd7bf4c2b80ca2ac9ac35bbbe as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/B/9cfe360cd7bf4c2b80ca2ac9ac35bbbe 2024-11-20T09:00:54,643 INFO [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/B/9cfe360cd7bf4c2b80ca2ac9ac35bbbe, entries=100, sequenceid=432, filesize=9.6 K 2024-11-20T09:00:54,644 DEBUG [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/.tmp/C/2e27a144a4274299aab39b51b6f591fb as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/C/2e27a144a4274299aab39b51b6f591fb 2024-11-20T09:00:54,646 INFO [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/C/2e27a144a4274299aab39b51b6f591fb, entries=100, sequenceid=432, filesize=9.6 K 2024-11-20T09:00:54,647 INFO [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~53.44 KB/54720, currentSize=0 B/0 for 11f6fecb00005c6636e7ea541ff70dfb in 1241ms, sequenceid=432, compaction requested=true 2024-11-20T09:00:54,648 DEBUG [StoreCloser-TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/A/1b201078cdb94e56b1f2af61c2f11a4e, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/A/ebc60d666f4a4ea7ba605455f377dcdc, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/A/e909f16849d049d08d098ae69809ddeb, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/A/9ef3f27fdf994e5988897c44285ba1ec, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/A/30f9fa05ff3945e29019e13f780b9b49, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/A/15806525791e4e58818dd7d6c9dfa5df, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/A/c49d7d2f92d94144bfbbcc3f60c6a206, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/A/8846273420cc4c68bd0ecddfafe4fc5d, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/A/aa33f71a11f44f7f925287662e82ba4f, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/A/1440badfcf3445b0b61e9d33ab2e6d07, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/A/1a19596e89ba4a688090ddbdb0d02951, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/A/48d9cb2dc3194b68bfed2766b550c0c4, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/A/78bd4f6bb0254ae586740084a776407f, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/A/7b478c09b1004a45a5f4d2d89596c235, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/A/5f951ec18f604abeb651fd0e5ec4bf7c, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/A/411dcbf93be246be96977c2bc8d25cdf, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/A/1923e25e14774215b677c209bfc5b85e, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/A/48a62c29d37d4ba09b8f065e302acb5e, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/A/b2d6779930e549af96d69528268996b3, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/A/024c298818824097abc1d147b1cc1038, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/A/9c6ee43cfb2e46e697bdfdcdf42709c1, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/A/1e822da94f664d80b56c0667f388e38a, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/A/67f870e81e784985b46cfb1c00573123, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/A/aace4f2251ad4faab88ec64b5cb0ec6f, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/A/34901012e07c4467979474a7c63f5f96, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/A/6a7608898d4748d4b8992f173271816a, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/A/c9fc2e03580a4e238c485eb9a465bb6b, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/A/1ae95256b2ad45329c92eede78d6e983, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/A/49409cd49b8349b0b1b931753cc719c3, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/A/d80088a4392f44bf8749b15543f6b3f0] to archive 2024-11-20T09:00:54,648 DEBUG [StoreCloser-TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-20T09:00:54,650 DEBUG [StoreCloser-TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/A/1b201078cdb94e56b1f2af61c2f11a4e to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/A/1b201078cdb94e56b1f2af61c2f11a4e 2024-11-20T09:00:54,650 DEBUG [StoreCloser-TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/A/ebc60d666f4a4ea7ba605455f377dcdc to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/A/ebc60d666f4a4ea7ba605455f377dcdc 2024-11-20T09:00:54,651 DEBUG [StoreCloser-TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/A/e909f16849d049d08d098ae69809ddeb to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/A/e909f16849d049d08d098ae69809ddeb 2024-11-20T09:00:54,652 DEBUG [StoreCloser-TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/A/9ef3f27fdf994e5988897c44285ba1ec to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/A/9ef3f27fdf994e5988897c44285ba1ec 2024-11-20T09:00:54,653 DEBUG [StoreCloser-TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/A/30f9fa05ff3945e29019e13f780b9b49 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/A/30f9fa05ff3945e29019e13f780b9b49 2024-11-20T09:00:54,654 DEBUG [StoreCloser-TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/A/15806525791e4e58818dd7d6c9dfa5df to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/A/15806525791e4e58818dd7d6c9dfa5df 2024-11-20T09:00:54,654 DEBUG [StoreCloser-TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/A/c49d7d2f92d94144bfbbcc3f60c6a206 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/A/c49d7d2f92d94144bfbbcc3f60c6a206 2024-11-20T09:00:54,655 DEBUG [StoreCloser-TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/A/8846273420cc4c68bd0ecddfafe4fc5d to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/A/8846273420cc4c68bd0ecddfafe4fc5d 2024-11-20T09:00:54,656 DEBUG [StoreCloser-TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/A/aa33f71a11f44f7f925287662e82ba4f to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/A/aa33f71a11f44f7f925287662e82ba4f 2024-11-20T09:00:54,657 DEBUG [StoreCloser-TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/A/1440badfcf3445b0b61e9d33ab2e6d07 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/A/1440badfcf3445b0b61e9d33ab2e6d07 2024-11-20T09:00:54,658 DEBUG [StoreCloser-TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/A/1a19596e89ba4a688090ddbdb0d02951 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/A/1a19596e89ba4a688090ddbdb0d02951 2024-11-20T09:00:54,659 DEBUG [StoreCloser-TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/A/48d9cb2dc3194b68bfed2766b550c0c4 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/A/48d9cb2dc3194b68bfed2766b550c0c4 2024-11-20T09:00:54,659 DEBUG [StoreCloser-TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/A/78bd4f6bb0254ae586740084a776407f to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/A/78bd4f6bb0254ae586740084a776407f 2024-11-20T09:00:54,660 DEBUG [StoreCloser-TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/A/7b478c09b1004a45a5f4d2d89596c235 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/A/7b478c09b1004a45a5f4d2d89596c235 2024-11-20T09:00:54,661 DEBUG [StoreCloser-TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/A/5f951ec18f604abeb651fd0e5ec4bf7c to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/A/5f951ec18f604abeb651fd0e5ec4bf7c 2024-11-20T09:00:54,662 DEBUG [StoreCloser-TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/A/411dcbf93be246be96977c2bc8d25cdf to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/A/411dcbf93be246be96977c2bc8d25cdf 2024-11-20T09:00:54,663 DEBUG [StoreCloser-TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/A/1923e25e14774215b677c209bfc5b85e to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/A/1923e25e14774215b677c209bfc5b85e 2024-11-20T09:00:54,663 DEBUG [StoreCloser-TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/A/48a62c29d37d4ba09b8f065e302acb5e to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/A/48a62c29d37d4ba09b8f065e302acb5e 2024-11-20T09:00:54,664 DEBUG [StoreCloser-TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/A/b2d6779930e549af96d69528268996b3 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/A/b2d6779930e549af96d69528268996b3 2024-11-20T09:00:54,665 DEBUG [StoreCloser-TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/A/024c298818824097abc1d147b1cc1038 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/A/024c298818824097abc1d147b1cc1038 2024-11-20T09:00:54,666 DEBUG [StoreCloser-TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/A/9c6ee43cfb2e46e697bdfdcdf42709c1 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/A/9c6ee43cfb2e46e697bdfdcdf42709c1 2024-11-20T09:00:54,667 DEBUG [StoreCloser-TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/A/1e822da94f664d80b56c0667f388e38a to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/A/1e822da94f664d80b56c0667f388e38a 2024-11-20T09:00:54,668 DEBUG [StoreCloser-TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/A/67f870e81e784985b46cfb1c00573123 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/A/67f870e81e784985b46cfb1c00573123 2024-11-20T09:00:54,668 DEBUG [StoreCloser-TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/A/aace4f2251ad4faab88ec64b5cb0ec6f to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/A/aace4f2251ad4faab88ec64b5cb0ec6f 2024-11-20T09:00:54,669 DEBUG [StoreCloser-TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/A/34901012e07c4467979474a7c63f5f96 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/A/34901012e07c4467979474a7c63f5f96 2024-11-20T09:00:54,670 DEBUG [StoreCloser-TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/A/6a7608898d4748d4b8992f173271816a to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/A/6a7608898d4748d4b8992f173271816a 2024-11-20T09:00:54,671 DEBUG [StoreCloser-TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/A/c9fc2e03580a4e238c485eb9a465bb6b to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/A/c9fc2e03580a4e238c485eb9a465bb6b 2024-11-20T09:00:54,672 DEBUG [StoreCloser-TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/A/1ae95256b2ad45329c92eede78d6e983 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/A/1ae95256b2ad45329c92eede78d6e983 2024-11-20T09:00:54,673 DEBUG [StoreCloser-TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/A/49409cd49b8349b0b1b931753cc719c3 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/A/49409cd49b8349b0b1b931753cc719c3 2024-11-20T09:00:54,673 DEBUG [StoreCloser-TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/A/d80088a4392f44bf8749b15543f6b3f0 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/A/d80088a4392f44bf8749b15543f6b3f0 2024-11-20T09:00:54,675 DEBUG [StoreCloser-TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/B/c5c0794e81dc4f188b8762191241dae0, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/B/f4b3551841694e33a70f3f73315f4584, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/B/2e6863d840a64ee4aacf05869b593f42, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/B/7b28b22d5df941858c7448a22c88959a, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/B/3de3f7d0a4214a028ed7fd0dd3494909, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/B/a0bf154b946d49c28b6494d362a65beb, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/B/a4693a46097b44c2a65f205f76d6f475, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/B/4364ee9b16e64c13a4cd14451a912e16, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/B/10c856ff67864213877f268d3471a45d, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/B/f861033725bf4d5091d5919c59a2c43a, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/B/e69d5c8cfe76497eaceff71cc78851f5, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/B/9fc341a9aef846c1817248db10621a73, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/B/96ee158b8ad54cbfa83fed3c3e554602, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/B/8746a2c44aa04cbaaf775c28d8fe006e, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/B/7968deabefaa43908a63ff4a07bc04c4, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/B/11b0cbe124f846db8ef755b72e55d758, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/B/f97fc93675a141cabe97a8bb7d72cb10, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/B/97960baeee5d42998d385d9c3eb18b6c, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/B/f647fc6a8f6146a2adbf52b2a5450542, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/B/126b6932c41a4686930eb400ecc3eb71, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/B/8445dfb0fd5844dc8c27f8fe57383943, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/B/335ef6366f924d7aaa59cf5d394ebdc7, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/B/5282e2d05ada4e18a5d053ffc37f5e30, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/B/cfc15da4f79a4f7e896a32b1b58b42cf, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/B/40ceb6a99520419cbdcd82b34ef9a008, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/B/cac11e6925994a06a04895620fe116a8, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/B/8a34e27ac2354128a23e381666fb6a6f, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/B/09b4a71772ed4dc0a0bb7bf0bfba96c0, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/B/4dd0cc50724e4e11b7800db81dda6307, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/B/7149235b10d54d84b3f73e3ecac84707] to archive 2024-11-20T09:00:54,676 DEBUG [StoreCloser-TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-20T09:00:54,677 DEBUG [StoreCloser-TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/B/c5c0794e81dc4f188b8762191241dae0 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/B/c5c0794e81dc4f188b8762191241dae0 2024-11-20T09:00:54,678 DEBUG [StoreCloser-TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/B/f4b3551841694e33a70f3f73315f4584 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/B/f4b3551841694e33a70f3f73315f4584 2024-11-20T09:00:54,679 DEBUG [StoreCloser-TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/B/2e6863d840a64ee4aacf05869b593f42 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/B/2e6863d840a64ee4aacf05869b593f42 2024-11-20T09:00:54,679 DEBUG [StoreCloser-TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/B/7b28b22d5df941858c7448a22c88959a to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/B/7b28b22d5df941858c7448a22c88959a 2024-11-20T09:00:54,680 DEBUG [StoreCloser-TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/B/3de3f7d0a4214a028ed7fd0dd3494909 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/B/3de3f7d0a4214a028ed7fd0dd3494909 2024-11-20T09:00:54,681 DEBUG [StoreCloser-TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/B/a0bf154b946d49c28b6494d362a65beb to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/B/a0bf154b946d49c28b6494d362a65beb 2024-11-20T09:00:54,682 DEBUG [StoreCloser-TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/B/a4693a46097b44c2a65f205f76d6f475 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/B/a4693a46097b44c2a65f205f76d6f475 2024-11-20T09:00:54,683 DEBUG [StoreCloser-TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/B/4364ee9b16e64c13a4cd14451a912e16 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/B/4364ee9b16e64c13a4cd14451a912e16 2024-11-20T09:00:54,684 DEBUG [StoreCloser-TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/B/10c856ff67864213877f268d3471a45d to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/B/10c856ff67864213877f268d3471a45d 2024-11-20T09:00:54,685 DEBUG [StoreCloser-TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/B/f861033725bf4d5091d5919c59a2c43a to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/B/f861033725bf4d5091d5919c59a2c43a 2024-11-20T09:00:54,686 DEBUG [StoreCloser-TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/B/e69d5c8cfe76497eaceff71cc78851f5 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/B/e69d5c8cfe76497eaceff71cc78851f5 2024-11-20T09:00:54,687 DEBUG [StoreCloser-TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/B/9fc341a9aef846c1817248db10621a73 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/B/9fc341a9aef846c1817248db10621a73 2024-11-20T09:00:54,688 DEBUG [StoreCloser-TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/B/96ee158b8ad54cbfa83fed3c3e554602 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/B/96ee158b8ad54cbfa83fed3c3e554602 2024-11-20T09:00:54,688 DEBUG [StoreCloser-TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/B/8746a2c44aa04cbaaf775c28d8fe006e to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/B/8746a2c44aa04cbaaf775c28d8fe006e 2024-11-20T09:00:54,689 DEBUG [StoreCloser-TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/B/7968deabefaa43908a63ff4a07bc04c4 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/B/7968deabefaa43908a63ff4a07bc04c4 2024-11-20T09:00:54,690 DEBUG [StoreCloser-TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/B/11b0cbe124f846db8ef755b72e55d758 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/B/11b0cbe124f846db8ef755b72e55d758 2024-11-20T09:00:54,691 DEBUG [StoreCloser-TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/B/f97fc93675a141cabe97a8bb7d72cb10 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/B/f97fc93675a141cabe97a8bb7d72cb10 2024-11-20T09:00:54,692 DEBUG [StoreCloser-TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/B/97960baeee5d42998d385d9c3eb18b6c to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/B/97960baeee5d42998d385d9c3eb18b6c 2024-11-20T09:00:54,692 DEBUG [StoreCloser-TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/B/f647fc6a8f6146a2adbf52b2a5450542 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/B/f647fc6a8f6146a2adbf52b2a5450542 2024-11-20T09:00:54,693 DEBUG [StoreCloser-TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/B/126b6932c41a4686930eb400ecc3eb71 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/B/126b6932c41a4686930eb400ecc3eb71 2024-11-20T09:00:54,694 DEBUG [StoreCloser-TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/B/8445dfb0fd5844dc8c27f8fe57383943 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/B/8445dfb0fd5844dc8c27f8fe57383943 2024-11-20T09:00:54,695 DEBUG [StoreCloser-TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/B/335ef6366f924d7aaa59cf5d394ebdc7 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/B/335ef6366f924d7aaa59cf5d394ebdc7 2024-11-20T09:00:54,696 DEBUG [StoreCloser-TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/B/5282e2d05ada4e18a5d053ffc37f5e30 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/B/5282e2d05ada4e18a5d053ffc37f5e30 2024-11-20T09:00:54,696 DEBUG [StoreCloser-TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/B/cfc15da4f79a4f7e896a32b1b58b42cf to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/B/cfc15da4f79a4f7e896a32b1b58b42cf 2024-11-20T09:00:54,697 DEBUG [StoreCloser-TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/B/40ceb6a99520419cbdcd82b34ef9a008 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/B/40ceb6a99520419cbdcd82b34ef9a008 2024-11-20T09:00:54,698 DEBUG [StoreCloser-TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/B/cac11e6925994a06a04895620fe116a8 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/B/cac11e6925994a06a04895620fe116a8 2024-11-20T09:00:54,699 DEBUG [StoreCloser-TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/B/8a34e27ac2354128a23e381666fb6a6f to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/B/8a34e27ac2354128a23e381666fb6a6f 2024-11-20T09:00:54,700 DEBUG [StoreCloser-TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/B/09b4a71772ed4dc0a0bb7bf0bfba96c0 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/B/09b4a71772ed4dc0a0bb7bf0bfba96c0 2024-11-20T09:00:54,701 DEBUG [StoreCloser-TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/B/4dd0cc50724e4e11b7800db81dda6307 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/B/4dd0cc50724e4e11b7800db81dda6307 2024-11-20T09:00:54,702 DEBUG [StoreCloser-TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/B/7149235b10d54d84b3f73e3ecac84707 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/B/7149235b10d54d84b3f73e3ecac84707 2024-11-20T09:00:54,703 DEBUG [StoreCloser-TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/C/93564b184a244cfd964dc54e1e3b1625, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/C/9de830fc02404ecf9b1048b24acdf5b5, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/C/b463a22430364744b162d403b2d380a7, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/C/628fb7b69bdb44d894710bf71ed5a1c4, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/C/b423d82390304e08a6b1ca9fc8eecdb0, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/C/b3162f2c44a14e33a260e7c69d9c0c71, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/C/7e28c302b3814e61b87aa454c218dc1c, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/C/83d7f187dcb54ec78e6ec93f80690f58, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/C/d87ce43d0b734a4480a03c4a97fd5d0e, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/C/56bead7fe4f54643b4ad9bc6fd45b21d, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/C/f7ee7238225e4aa5a0e87ed4cf214210, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/C/d97e784a8add44859c070d9720afd60d, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/C/0c582d7f578b4e83bae3ac21f218d268, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/C/79f06b0b6eff4d8e82a8c9b337f84cfd, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/C/ab7ee26379784aafa2ac4eb0173d2f2a, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/C/510e463161fc4fc8937b29097c47acd4, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/C/c38cf285c47348c493ba151b8373d971, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/C/5ef39d459ba1454eaa6e167b854acf45, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/C/7a1697d12aae47b7a4eef2c665800f1f, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/C/6cb47df77d87481283529e7145462664, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/C/1584ebbccfa740d7add5768fe6f139a9, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/C/a106995fc55f44489ae2b67fe8ab49d7, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/C/524eda594786446aa31478f1bfa1fdc9, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/C/b9b9ac335bbd4fa09be2f104acfd1fcc, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/C/b66527f898c74de8b1c64fcd0befa3cf, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/C/3ecf0ab1108f4dfba126366c348adbc7] to archive 2024-11-20T09:00:54,704 DEBUG [StoreCloser-TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-20T09:00:54,705 DEBUG [StoreCloser-TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/C/93564b184a244cfd964dc54e1e3b1625 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/C/93564b184a244cfd964dc54e1e3b1625 2024-11-20T09:00:54,706 DEBUG [StoreCloser-TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/C/9de830fc02404ecf9b1048b24acdf5b5 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/C/9de830fc02404ecf9b1048b24acdf5b5 2024-11-20T09:00:54,707 DEBUG [StoreCloser-TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/C/b463a22430364744b162d403b2d380a7 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/C/b463a22430364744b162d403b2d380a7 2024-11-20T09:00:54,708 DEBUG [StoreCloser-TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/C/628fb7b69bdb44d894710bf71ed5a1c4 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/C/628fb7b69bdb44d894710bf71ed5a1c4 2024-11-20T09:00:54,708 DEBUG [StoreCloser-TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/C/b423d82390304e08a6b1ca9fc8eecdb0 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/C/b423d82390304e08a6b1ca9fc8eecdb0 2024-11-20T09:00:54,709 DEBUG [StoreCloser-TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/C/b3162f2c44a14e33a260e7c69d9c0c71 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/C/b3162f2c44a14e33a260e7c69d9c0c71 2024-11-20T09:00:54,710 DEBUG [StoreCloser-TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/C/7e28c302b3814e61b87aa454c218dc1c to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/C/7e28c302b3814e61b87aa454c218dc1c 2024-11-20T09:00:54,711 DEBUG [StoreCloser-TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/C/83d7f187dcb54ec78e6ec93f80690f58 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/C/83d7f187dcb54ec78e6ec93f80690f58 2024-11-20T09:00:54,711 DEBUG [StoreCloser-TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/C/d87ce43d0b734a4480a03c4a97fd5d0e to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/C/d87ce43d0b734a4480a03c4a97fd5d0e 2024-11-20T09:00:54,712 DEBUG [StoreCloser-TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/C/56bead7fe4f54643b4ad9bc6fd45b21d to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/C/56bead7fe4f54643b4ad9bc6fd45b21d 2024-11-20T09:00:54,713 DEBUG [StoreCloser-TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/C/f7ee7238225e4aa5a0e87ed4cf214210 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/C/f7ee7238225e4aa5a0e87ed4cf214210 2024-11-20T09:00:54,714 DEBUG [StoreCloser-TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/C/d97e784a8add44859c070d9720afd60d to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/C/d97e784a8add44859c070d9720afd60d 2024-11-20T09:00:54,715 DEBUG [StoreCloser-TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/C/0c582d7f578b4e83bae3ac21f218d268 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/C/0c582d7f578b4e83bae3ac21f218d268 2024-11-20T09:00:54,715 DEBUG [StoreCloser-TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/C/79f06b0b6eff4d8e82a8c9b337f84cfd to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/C/79f06b0b6eff4d8e82a8c9b337f84cfd 2024-11-20T09:00:54,716 DEBUG [StoreCloser-TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/C/ab7ee26379784aafa2ac4eb0173d2f2a to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/C/ab7ee26379784aafa2ac4eb0173d2f2a 2024-11-20T09:00:54,717 DEBUG [StoreCloser-TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/C/510e463161fc4fc8937b29097c47acd4 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/C/510e463161fc4fc8937b29097c47acd4 2024-11-20T09:00:54,718 DEBUG [StoreCloser-TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/C/c38cf285c47348c493ba151b8373d971 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/C/c38cf285c47348c493ba151b8373d971 2024-11-20T09:00:54,718 DEBUG [StoreCloser-TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/C/5ef39d459ba1454eaa6e167b854acf45 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/C/5ef39d459ba1454eaa6e167b854acf45 2024-11-20T09:00:54,719 DEBUG [StoreCloser-TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/C/7a1697d12aae47b7a4eef2c665800f1f to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/C/7a1697d12aae47b7a4eef2c665800f1f 2024-11-20T09:00:54,720 DEBUG [StoreCloser-TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/C/6cb47df77d87481283529e7145462664 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/C/6cb47df77d87481283529e7145462664 2024-11-20T09:00:54,721 DEBUG [StoreCloser-TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/C/1584ebbccfa740d7add5768fe6f139a9 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/C/1584ebbccfa740d7add5768fe6f139a9 2024-11-20T09:00:54,721 DEBUG [StoreCloser-TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/C/a106995fc55f44489ae2b67fe8ab49d7 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/C/a106995fc55f44489ae2b67fe8ab49d7 2024-11-20T09:00:54,722 DEBUG [StoreCloser-TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/C/524eda594786446aa31478f1bfa1fdc9 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/C/524eda594786446aa31478f1bfa1fdc9 2024-11-20T09:00:54,723 DEBUG [StoreCloser-TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/C/b9b9ac335bbd4fa09be2f104acfd1fcc to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/C/b9b9ac335bbd4fa09be2f104acfd1fcc 2024-11-20T09:00:54,723 DEBUG [StoreCloser-TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/C/b66527f898c74de8b1c64fcd0befa3cf to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/C/b66527f898c74de8b1c64fcd0befa3cf 2024-11-20T09:00:54,724 DEBUG [StoreCloser-TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/C/3ecf0ab1108f4dfba126366c348adbc7 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/C/3ecf0ab1108f4dfba126366c348adbc7 2024-11-20T09:00:54,728 DEBUG [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/recovered.edits/435.seqid, newMaxSeqId=435, maxSeqId=1 2024-11-20T09:00:54,729 INFO [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb. 2024-11-20T09:00:54,729 DEBUG [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1676): Region close journal for 11f6fecb00005c6636e7ea541ff70dfb: Waiting for close lock at 1732093253221Running coprocessor pre-close hooks at 1732093253221Disabling compacts and flushes for region at 1732093253221Disabling writes for close at 1732093253406 (+185 ms)Obtaining lock to block concurrent updates at 1732093253406Preparing flush snapshotting stores in 11f6fecb00005c6636e7ea541ff70dfb at 1732093253406Finished memstore snapshotting TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb., syncing WAL and waiting on mvcc, flushsize=dataSize=20610, getHeapSize=54720, getOffHeapSize=0, getCellsCount=450 at 1732093253407 (+1 ms)Flushing stores of TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb. at 1732093253407Flushing 11f6fecb00005c6636e7ea541ff70dfb/A: creating writer at 1732093253407Flushing 11f6fecb00005c6636e7ea541ff70dfb/A: appending metadata at 1732093253409 (+2 ms)Flushing 11f6fecb00005c6636e7ea541ff70dfb/A: closing flushed file at 1732093253409Flushing 11f6fecb00005c6636e7ea541ff70dfb/B: creating writer at 1732093253816 (+407 ms)Flushing 11f6fecb00005c6636e7ea541ff70dfb/B: appending metadata at 1732093253819 (+3 ms)Flushing 11f6fecb00005c6636e7ea541ff70dfb/B: closing flushed file at 1732093253819Flushing 11f6fecb00005c6636e7ea541ff70dfb/C: creating writer at 1732093254226 (+407 ms)Flushing 11f6fecb00005c6636e7ea541ff70dfb/C: appending metadata at 1732093254229 (+3 ms)Flushing 11f6fecb00005c6636e7ea541ff70dfb/C: closing flushed file at 1732093254229Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@147e1332: reopening flushed file at 1732093254636 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5e14c8e6: reopening flushed file at 1732093254640 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@293d6f02: reopening flushed file at 1732093254643 (+3 ms)Finished flush of dataSize ~20.13 KB/20610, heapSize ~53.44 KB/54720, currentSize=0 B/0 for 11f6fecb00005c6636e7ea541ff70dfb in 1241ms, sequenceid=432, compaction requested=true at 1732093254647 (+4 ms)Writing region close event to WAL at 1732093254725 (+78 ms)Running coprocessor post-close hooks at 1732093254729 (+4 ms)Closed at 1732093254729 2024-11-20T09:00:54,731 INFO [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] handler.UnassignRegionHandler(157): Closed 11f6fecb00005c6636e7ea541ff70dfb 2024-11-20T09:00:54,731 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=82 updating hbase:meta row=11f6fecb00005c6636e7ea541ff70dfb, regionState=CLOSED 2024-11-20T09:00:54,732 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=83, ppid=82, state=RUNNABLE, hasLock=false; CloseRegionProcedure 11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 because future has completed 2024-11-20T09:00:54,735 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=83, resume processing ppid=82 2024-11-20T09:00:54,735 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=83, ppid=82, state=SUCCESS, hasLock=false; CloseRegionProcedure 11f6fecb00005c6636e7ea541ff70dfb, server=be1eb2620e0e,34223,1732093159960 in 1.6640 sec 2024-11-20T09:00:54,736 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=82, resume processing ppid=81 2024-11-20T09:00:54,736 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=82, ppid=81, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=11f6fecb00005c6636e7ea541ff70dfb, UNASSIGN in 1.6690 sec 2024-11-20T09:00:54,738 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=81, resume processing ppid=80 2024-11-20T09:00:54,738 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, ppid=80, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.6720 sec 2024-11-20T09:00:54,739 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732093254739"}]},"ts":"1732093254739"} 2024-11-20T09:00:54,740 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-20T09:00:54,740 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-20T09:00:54,741 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.6810 sec 2024-11-20T09:00:55,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-20T09:00:55,207 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-20T09:00:55,208 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-20T09:00:55,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] procedure2.ProcedureExecutor(1139): Stored pid=84, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T09:00:55,209 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=84, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T09:00:55,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-20T09:00:55,210 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=84, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T09:00:55,212 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb 2024-11-20T09:00:55,214 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/A, FileablePath, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/B, FileablePath, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/C, FileablePath, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/recovered.edits] 2024-11-20T09:00:55,216 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/A/cff9586e8f774ef6b59bff9ef3373bdd to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/A/cff9586e8f774ef6b59bff9ef3373bdd 2024-11-20T09:00:55,217 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/A/f13c9520bb414850a8a4427d0212ccb6 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/A/f13c9520bb414850a8a4427d0212ccb6 2024-11-20T09:00:55,219 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/B/9cfe360cd7bf4c2b80ca2ac9ac35bbbe to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/B/9cfe360cd7bf4c2b80ca2ac9ac35bbbe 2024-11-20T09:00:55,220 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/B/bbf8d66daece4f8faeeca715cfa86914 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/B/bbf8d66daece4f8faeeca715cfa86914 2024-11-20T09:00:55,222 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/C/2e27a144a4274299aab39b51b6f591fb to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/C/2e27a144a4274299aab39b51b6f591fb 2024-11-20T09:00:55,223 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/C/3a8e6cdb12054c2a9889257c61f91277 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/C/3a8e6cdb12054c2a9889257c61f91277 2024-11-20T09:00:55,224 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/C/523cc90a6c3b4dc3ae3e9f6eec68e5c3 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/C/523cc90a6c3b4dc3ae3e9f6eec68e5c3 2024-11-20T09:00:55,225 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/C/b9a10ef5c89245b7be4a6c787210f3f9 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/C/b9a10ef5c89245b7be4a6c787210f3f9 2024-11-20T09:00:55,225 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/C/fecb2746074648678b0f8a2336fa69fe to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/C/fecb2746074648678b0f8a2336fa69fe 2024-11-20T09:00:55,228 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/recovered.edits/435.seqid to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb/recovered.edits/435.seqid 2024-11-20T09:00:55,228 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/11f6fecb00005c6636e7ea541ff70dfb 2024-11-20T09:00:55,228 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-20T09:00:55,230 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=84, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T09:00:55,232 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-20T09:00:55,234 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-20T09:00:55,235 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=84, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T09:00:55,235 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-20T09:00:55,235 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732093255235"}]},"ts":"9223372036854775807"} 2024-11-20T09:00:55,237 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-20T09:00:55,237 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 11f6fecb00005c6636e7ea541ff70dfb, NAME => 'TestAcidGuarantees,,1732093228786.11f6fecb00005c6636e7ea541ff70dfb.', STARTKEY => '', ENDKEY => ''}] 2024-11-20T09:00:55,237 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-20T09:00:55,237 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732093255237"}]},"ts":"9223372036854775807"} 2024-11-20T09:00:55,238 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-20T09:00:55,239 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=84, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T09:00:55,240 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=84, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 31 msec 2024-11-20T09:00:55,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-20T09:00:55,317 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-20T09:00:55,319 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-20T09:00:55,327 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testGetAtomicity Thread=242 (was 239) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_2065734148_22 at /127.0.0.1:40876 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_2065734148_22 at /127.0.0.1:38046 [Waiting for operation #246] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_656394272_22 at /127.0.0.1:55580 [Waiting for operation #170] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_656394272_22 at /127.0.0.1:55584 [Waiting for operation #168] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=459 (was 455) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=453 (was 486), ProcessCount=11 (was 11), AvailableMemoryMB=2041 (was 2143) 2024-11-20T09:00:55,335 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobScanAtomicity Thread=242, OpenFileDescriptor=459, MaxFileDescriptor=1048576, SystemLoadAverage=453, ProcessCount=11, AvailableMemoryMB=2041 2024-11-20T09:00:55,336 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-20T09:00:55,336 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-20T09:00:55,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] procedure2.ProcedureExecutor(1139): Stored pid=85, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-20T09:00:55,338 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=85, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-20T09:00:55,338 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:55,338 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 85 2024-11-20T09:00:55,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-11-20T09:00:55,339 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=85, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-20T09:00:55,344 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742202_1378 (size=959) 2024-11-20T09:00:55,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-11-20T09:00:55,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-11-20T09:00:55,746 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => b3cbab3048a5449de36008b3c0424a49, NAME => 'TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a 2024-11-20T09:00:55,750 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742203_1379 (size=53) 2024-11-20T09:00:55,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-11-20T09:00:56,151 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T09:00:56,151 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing b3cbab3048a5449de36008b3c0424a49, disabling compactions & flushes 2024-11-20T09:00:56,151 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49. 2024-11-20T09:00:56,151 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49. 2024-11-20T09:00:56,151 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49. after waiting 0 ms 2024-11-20T09:00:56,151 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49. 2024-11-20T09:00:56,151 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49. 2024-11-20T09:00:56,151 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for b3cbab3048a5449de36008b3c0424a49: Waiting for close lock at 1732093256151Disabling compacts and flushes for region at 1732093256151Disabling writes for close at 1732093256151Writing region close event to WAL at 1732093256151Closed at 1732093256151 2024-11-20T09:00:56,152 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=85, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-20T09:00:56,153 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732093256152"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732093256152"}]},"ts":"1732093256152"} 2024-11-20T09:00:56,154 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-20T09:00:56,155 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=85, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-20T09:00:56,155 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732093256155"}]},"ts":"1732093256155"} 2024-11-20T09:00:56,156 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-20T09:00:56,156 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b3cbab3048a5449de36008b3c0424a49, ASSIGN}] 2024-11-20T09:00:56,157 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b3cbab3048a5449de36008b3c0424a49, ASSIGN 2024-11-20T09:00:56,158 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=b3cbab3048a5449de36008b3c0424a49, ASSIGN; state=OFFLINE, location=be1eb2620e0e,34223,1732093159960; forceNewPlan=false, retain=false 2024-11-20T09:00:56,308 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=86 updating hbase:meta row=b3cbab3048a5449de36008b3c0424a49, regionState=OPENING, regionLocation=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:56,310 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=b3cbab3048a5449de36008b3c0424a49, ASSIGN because future has completed 2024-11-20T09:00:56,310 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=87, ppid=86, state=RUNNABLE, hasLock=false; OpenRegionProcedure b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960}] 2024-11-20T09:00:56,464 INFO [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49. 2024-11-20T09:00:56,464 DEBUG [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(7752): Opening region: {ENCODED => b3cbab3048a5449de36008b3c0424a49, NAME => 'TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49.', STARTKEY => '', ENDKEY => ''} 2024-11-20T09:00:56,465 DEBUG [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees b3cbab3048a5449de36008b3c0424a49 2024-11-20T09:00:56,465 DEBUG [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T09:00:56,465 DEBUG [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(7794): checking encryption for b3cbab3048a5449de36008b3c0424a49 2024-11-20T09:00:56,465 DEBUG [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(7797): checking classloading for b3cbab3048a5449de36008b3c0424a49 2024-11-20T09:00:56,466 INFO [StoreOpener-b3cbab3048a5449de36008b3c0424a49-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region b3cbab3048a5449de36008b3c0424a49 2024-11-20T09:00:56,467 INFO [StoreOpener-b3cbab3048a5449de36008b3c0424a49-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b3cbab3048a5449de36008b3c0424a49 columnFamilyName A 2024-11-20T09:00:56,467 DEBUG [StoreOpener-b3cbab3048a5449de36008b3c0424a49-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:56,468 INFO [StoreOpener-b3cbab3048a5449de36008b3c0424a49-1 {}] regionserver.HStore(327): Store=b3cbab3048a5449de36008b3c0424a49/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T09:00:56,468 INFO [StoreOpener-b3cbab3048a5449de36008b3c0424a49-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region b3cbab3048a5449de36008b3c0424a49 2024-11-20T09:00:56,469 INFO [StoreOpener-b3cbab3048a5449de36008b3c0424a49-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b3cbab3048a5449de36008b3c0424a49 columnFamilyName B 2024-11-20T09:00:56,469 DEBUG [StoreOpener-b3cbab3048a5449de36008b3c0424a49-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:56,469 INFO [StoreOpener-b3cbab3048a5449de36008b3c0424a49-1 {}] regionserver.HStore(327): Store=b3cbab3048a5449de36008b3c0424a49/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T09:00:56,469 INFO [StoreOpener-b3cbab3048a5449de36008b3c0424a49-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region b3cbab3048a5449de36008b3c0424a49 2024-11-20T09:00:56,470 INFO [StoreOpener-b3cbab3048a5449de36008b3c0424a49-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b3cbab3048a5449de36008b3c0424a49 columnFamilyName C 2024-11-20T09:00:56,470 DEBUG [StoreOpener-b3cbab3048a5449de36008b3c0424a49-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:56,470 INFO [StoreOpener-b3cbab3048a5449de36008b3c0424a49-1 {}] regionserver.HStore(327): Store=b3cbab3048a5449de36008b3c0424a49/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T09:00:56,470 DEBUG [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1038): replaying wal for b3cbab3048a5449de36008b3c0424a49 2024-11-20T09:00:56,471 DEBUG [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49 2024-11-20T09:00:56,471 DEBUG [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49 2024-11-20T09:00:56,472 DEBUG [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1048): stopping wal replay for b3cbab3048a5449de36008b3c0424a49 2024-11-20T09:00:56,472 DEBUG [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1060): Cleaning up temporary data for b3cbab3048a5449de36008b3c0424a49 2024-11-20T09:00:56,472 DEBUG [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-20T09:00:56,473 DEBUG [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1093): writing seq id for b3cbab3048a5449de36008b3c0424a49 2024-11-20T09:00:56,474 DEBUG [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T09:00:56,475 INFO [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1114): Opened b3cbab3048a5449de36008b3c0424a49; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66222127, jitterRate=-0.01321341097354889}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-20T09:00:56,475 DEBUG [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1122): Running coprocessor post-open hooks for b3cbab3048a5449de36008b3c0424a49 2024-11-20T09:00:56,475 DEBUG [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1006): Region open journal for b3cbab3048a5449de36008b3c0424a49: Running coprocessor pre-open hook at 1732093256465Writing region info on filesystem at 1732093256465Initializing all the Stores at 1732093256465Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732093256465Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732093256466 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732093256466Cleaning up temporary data from old regions at 1732093256472 (+6 ms)Running coprocessor post-open hooks at 1732093256475 (+3 ms)Region opened successfully at 1732093256475 2024-11-20T09:00:56,476 INFO [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., pid=87, masterSystemTime=1732093256461 2024-11-20T09:00:56,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-11-20T09:00:56,478 DEBUG [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49. 2024-11-20T09:00:56,478 INFO [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49. 2024-11-20T09:00:56,478 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=86 updating hbase:meta row=b3cbab3048a5449de36008b3c0424a49, regionState=OPEN, openSeqNum=2, regionLocation=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:56,479 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=87, ppid=86, state=RUNNABLE, hasLock=false; OpenRegionProcedure b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 because future has completed 2024-11-20T09:00:56,481 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=87, resume processing ppid=86 2024-11-20T09:00:56,481 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=87, ppid=86, state=SUCCESS, hasLock=false; OpenRegionProcedure b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 in 170 msec 2024-11-20T09:00:56,483 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=86, resume processing ppid=85 2024-11-20T09:00:56,483 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=86, ppid=85, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b3cbab3048a5449de36008b3c0424a49, ASSIGN in 325 msec 2024-11-20T09:00:56,483 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=85, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-20T09:00:56,483 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732093256483"}]},"ts":"1732093256483"} 2024-11-20T09:00:56,484 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-20T09:00:56,485 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=85, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-20T09:00:56,486 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=85, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1490 sec 2024-11-20T09:00:57,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-11-20T09:00:57,487 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-20T09:00:57,488 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@67f5e8d7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T09:00:57,488 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request be1eb2620e0e,44373,-1 for getting cluster id 2024-11-20T09:00:57,488 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T09:00:57,489 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'caf50035-b19e-4cfd-90ea-dca559866882' 2024-11-20T09:00:57,489 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T09:00:57,489 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "caf50035-b19e-4cfd-90ea-dca559866882" 2024-11-20T09:00:57,489 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f54f6e1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T09:00:57,490 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [be1eb2620e0e,44373,-1] 2024-11-20T09:00:57,490 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T09:00:57,490 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T09:00:57,491 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40276, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T09:00:57,491 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5de70cc8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T09:00:57,491 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T09:00:57,492 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=be1eb2620e0e,34223,1732093159960, seqNum=-1] 2024-11-20T09:00:57,492 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T09:00:57,493 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54914, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T09:00:57,494 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-20T09:00:57,494 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is be1eb2620e0e,44373,1732093159150 2024-11-20T09:00:57,494 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@56f58aea 2024-11-20T09:00:57,494 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-20T09:00:57,495 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40292, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-20T09:00:57,496 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-20T09:00:57,496 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-20T09:00:57,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] procedure2.ProcedureExecutor(1139): Stored pid=88, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-20T09:00:57,506 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742204_1380 (size=995) 2024-11-20T09:00:57,908 DEBUG [PEWorker-1 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-11-20T09:00:57,908 INFO [PEWorker-1 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-11-20T09:00:57,910 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=89, ppid=88, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-20T09:00:57,912 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b3cbab3048a5449de36008b3c0424a49, REOPEN/MOVE}] 2024-11-20T09:00:57,913 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b3cbab3048a5449de36008b3c0424a49, REOPEN/MOVE 2024-11-20T09:00:57,913 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=90 updating hbase:meta row=b3cbab3048a5449de36008b3c0424a49, regionState=CLOSING, regionLocation=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:57,914 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=b3cbab3048a5449de36008b3c0424a49, REOPEN/MOVE because future has completed 2024-11-20T09:00:57,915 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-20T09:00:57,915 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=91, ppid=90, state=RUNNABLE, hasLock=false; CloseRegionProcedure b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960}] 2024-11-20T09:00:58,067 INFO [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] handler.UnassignRegionHandler(122): Close b3cbab3048a5449de36008b3c0424a49 2024-11-20T09:00:58,067 DEBUG [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-20T09:00:58,067 DEBUG [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1722): Closing b3cbab3048a5449de36008b3c0424a49, disabling compactions & flushes 2024-11-20T09:00:58,067 INFO [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49. 2024-11-20T09:00:58,067 DEBUG [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49. 2024-11-20T09:00:58,067 DEBUG [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49. after waiting 0 ms 2024-11-20T09:00:58,067 DEBUG [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49. 2024-11-20T09:00:58,070 DEBUG [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-20T09:00:58,071 INFO [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49. 2024-11-20T09:00:58,071 DEBUG [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1676): Region close journal for b3cbab3048a5449de36008b3c0424a49: Waiting for close lock at 1732093258067Running coprocessor pre-close hooks at 1732093258067Disabling compacts and flushes for region at 1732093258067Disabling writes for close at 1732093258067Writing region close event to WAL at 1732093258067Running coprocessor post-close hooks at 1732093258071 (+4 ms)Closed at 1732093258071 2024-11-20T09:00:58,071 WARN [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegionServer(3299): Not adding moved region record: b3cbab3048a5449de36008b3c0424a49 to self. 2024-11-20T09:00:58,072 INFO [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] handler.UnassignRegionHandler(157): Closed b3cbab3048a5449de36008b3c0424a49 2024-11-20T09:00:58,073 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=90 updating hbase:meta row=b3cbab3048a5449de36008b3c0424a49, regionState=CLOSED 2024-11-20T09:00:58,074 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=91, ppid=90, state=RUNNABLE, hasLock=false; CloseRegionProcedure b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 because future has completed 2024-11-20T09:00:58,076 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=91, resume processing ppid=90 2024-11-20T09:00:58,076 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=91, ppid=90, state=SUCCESS, hasLock=false; CloseRegionProcedure b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 in 160 msec 2024-11-20T09:00:58,077 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=b3cbab3048a5449de36008b3c0424a49, REOPEN/MOVE; state=CLOSED, location=be1eb2620e0e,34223,1732093159960; forceNewPlan=false, retain=true 2024-11-20T09:00:58,227 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=90 updating hbase:meta row=b3cbab3048a5449de36008b3c0424a49, regionState=OPENING, regionLocation=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:58,229 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=b3cbab3048a5449de36008b3c0424a49, REOPEN/MOVE because future has completed 2024-11-20T09:00:58,229 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=92, ppid=90, state=RUNNABLE, hasLock=false; OpenRegionProcedure b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960}] 2024-11-20T09:00:58,384 INFO [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49. 2024-11-20T09:00:58,384 DEBUG [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(7752): Opening region: {ENCODED => b3cbab3048a5449de36008b3c0424a49, NAME => 'TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49.', STARTKEY => '', ENDKEY => ''} 2024-11-20T09:00:58,384 DEBUG [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees b3cbab3048a5449de36008b3c0424a49 2024-11-20T09:00:58,384 DEBUG [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T09:00:58,384 DEBUG [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(7794): checking encryption for b3cbab3048a5449de36008b3c0424a49 2024-11-20T09:00:58,384 DEBUG [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(7797): checking classloading for b3cbab3048a5449de36008b3c0424a49 2024-11-20T09:00:58,385 INFO [StoreOpener-b3cbab3048a5449de36008b3c0424a49-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region b3cbab3048a5449de36008b3c0424a49 2024-11-20T09:00:58,386 INFO [StoreOpener-b3cbab3048a5449de36008b3c0424a49-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b3cbab3048a5449de36008b3c0424a49 columnFamilyName A 2024-11-20T09:00:58,387 DEBUG [StoreOpener-b3cbab3048a5449de36008b3c0424a49-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:58,388 INFO [StoreOpener-b3cbab3048a5449de36008b3c0424a49-1 {}] regionserver.HStore(327): Store=b3cbab3048a5449de36008b3c0424a49/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T09:00:58,388 INFO [StoreOpener-b3cbab3048a5449de36008b3c0424a49-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region b3cbab3048a5449de36008b3c0424a49 2024-11-20T09:00:58,388 INFO [StoreOpener-b3cbab3048a5449de36008b3c0424a49-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b3cbab3048a5449de36008b3c0424a49 columnFamilyName B 2024-11-20T09:00:58,388 DEBUG [StoreOpener-b3cbab3048a5449de36008b3c0424a49-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:58,389 INFO [StoreOpener-b3cbab3048a5449de36008b3c0424a49-1 {}] regionserver.HStore(327): Store=b3cbab3048a5449de36008b3c0424a49/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T09:00:58,389 INFO [StoreOpener-b3cbab3048a5449de36008b3c0424a49-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region b3cbab3048a5449de36008b3c0424a49 2024-11-20T09:00:58,389 INFO [StoreOpener-b3cbab3048a5449de36008b3c0424a49-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b3cbab3048a5449de36008b3c0424a49 columnFamilyName C 2024-11-20T09:00:58,389 DEBUG [StoreOpener-b3cbab3048a5449de36008b3c0424a49-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:58,390 INFO [StoreOpener-b3cbab3048a5449de36008b3c0424a49-1 {}] regionserver.HStore(327): Store=b3cbab3048a5449de36008b3c0424a49/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T09:00:58,390 DEBUG [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1038): replaying wal for b3cbab3048a5449de36008b3c0424a49 2024-11-20T09:00:58,390 DEBUG [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49 2024-11-20T09:00:58,391 DEBUG [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49 2024-11-20T09:00:58,392 DEBUG [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1048): stopping wal replay for b3cbab3048a5449de36008b3c0424a49 2024-11-20T09:00:58,392 DEBUG [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1060): Cleaning up temporary data for b3cbab3048a5449de36008b3c0424a49 2024-11-20T09:00:58,392 DEBUG [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-20T09:00:58,393 DEBUG [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1093): writing seq id for b3cbab3048a5449de36008b3c0424a49 2024-11-20T09:00:58,394 INFO [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1114): Opened b3cbab3048a5449de36008b3c0424a49; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69414777, jitterRate=0.03436078131198883}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-20T09:00:58,394 DEBUG [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1122): Running coprocessor post-open hooks for b3cbab3048a5449de36008b3c0424a49 2024-11-20T09:00:58,394 DEBUG [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1006): Region open journal for b3cbab3048a5449de36008b3c0424a49: Running coprocessor pre-open hook at 1732093258384Writing region info on filesystem at 1732093258384Initializing all the Stores at 1732093258385 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732093258385Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732093258385Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732093258385Cleaning up temporary data from old regions at 1732093258392 (+7 ms)Running coprocessor post-open hooks at 1732093258394 (+2 ms)Region opened successfully at 1732093258394 2024-11-20T09:00:58,395 INFO [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., pid=92, masterSystemTime=1732093258381 2024-11-20T09:00:58,397 DEBUG [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49. 2024-11-20T09:00:58,397 INFO [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49. 2024-11-20T09:00:58,397 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=90 updating hbase:meta row=b3cbab3048a5449de36008b3c0424a49, regionState=OPEN, openSeqNum=5, regionLocation=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:58,398 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=92, ppid=90, state=RUNNABLE, hasLock=false; OpenRegionProcedure b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 because future has completed 2024-11-20T09:00:58,400 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=92, resume processing ppid=90 2024-11-20T09:00:58,400 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=92, ppid=90, state=SUCCESS, hasLock=false; OpenRegionProcedure b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 in 170 msec 2024-11-20T09:00:58,402 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=90, resume processing ppid=89 2024-11-20T09:00:58,402 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=90, ppid=89, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b3cbab3048a5449de36008b3c0424a49, REOPEN/MOVE in 488 msec 2024-11-20T09:00:58,404 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=89, resume processing ppid=88 2024-11-20T09:00:58,404 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=89, ppid=88, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 492 msec 2024-11-20T09:00:58,406 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=88, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 907 msec 2024-11-20T09:00:58,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-20T09:00:58,406 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-20T09:00:58,407 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21c265f0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T09:00:58,407 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request be1eb2620e0e,44373,-1 for getting cluster id 2024-11-20T09:00:58,407 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T09:00:58,408 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'caf50035-b19e-4cfd-90ea-dca559866882' 2024-11-20T09:00:58,408 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T09:00:58,408 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "caf50035-b19e-4cfd-90ea-dca559866882" 2024-11-20T09:00:58,409 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64223b25, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T09:00:58,409 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [be1eb2620e0e,44373,-1] 2024-11-20T09:00:58,409 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T09:00:58,409 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T09:00:58,410 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40298, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T09:00:58,410 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74b1c05d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T09:00:58,411 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c21b87a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T09:00:58,411 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request be1eb2620e0e,44373,-1 for getting cluster id 2024-11-20T09:00:58,411 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T09:00:58,411 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'caf50035-b19e-4cfd-90ea-dca559866882' 2024-11-20T09:00:58,412 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T09:00:58,412 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "caf50035-b19e-4cfd-90ea-dca559866882" 2024-11-20T09:00:58,412 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ee2db18, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T09:00:58,412 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [be1eb2620e0e,44373,-1] 2024-11-20T09:00:58,412 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T09:00:58,412 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T09:00:58,413 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40312, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T09:00:58,413 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ba4cbfd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T09:00:58,414 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35e136ad, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T09:00:58,414 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request be1eb2620e0e,44373,-1 for getting cluster id 2024-11-20T09:00:58,414 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T09:00:58,414 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'caf50035-b19e-4cfd-90ea-dca559866882' 2024-11-20T09:00:58,414 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T09:00:58,414 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "caf50035-b19e-4cfd-90ea-dca559866882" 2024-11-20T09:00:58,414 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5acea2ed, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T09:00:58,414 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [be1eb2620e0e,44373,-1] 2024-11-20T09:00:58,415 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T09:00:58,415 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T09:00:58,415 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40328, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T09:00:58,416 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f5c99d4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T09:00:58,416 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f004f7c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T09:00:58,416 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request be1eb2620e0e,44373,-1 for getting cluster id 2024-11-20T09:00:58,416 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T09:00:58,417 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'caf50035-b19e-4cfd-90ea-dca559866882' 2024-11-20T09:00:58,417 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T09:00:58,417 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "caf50035-b19e-4cfd-90ea-dca559866882" 2024-11-20T09:00:58,417 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@147a8e0d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T09:00:58,417 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [be1eb2620e0e,44373,-1] 2024-11-20T09:00:58,417 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T09:00:58,417 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T09:00:58,418 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40350, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T09:00:58,418 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4eea00f0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T09:00:58,419 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@65616af9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T09:00:58,419 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request be1eb2620e0e,44373,-1 for getting cluster id 2024-11-20T09:00:58,419 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T09:00:58,419 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'caf50035-b19e-4cfd-90ea-dca559866882' 2024-11-20T09:00:58,420 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T09:00:58,420 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "caf50035-b19e-4cfd-90ea-dca559866882" 2024-11-20T09:00:58,420 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47d6a4d4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T09:00:58,420 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [be1eb2620e0e,44373,-1] 2024-11-20T09:00:58,420 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T09:00:58,420 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T09:00:58,421 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40366, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T09:00:58,421 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33110733, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T09:00:58,421 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@53f9edfd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T09:00:58,421 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request be1eb2620e0e,44373,-1 for getting cluster id 2024-11-20T09:00:58,422 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T09:00:58,422 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'caf50035-b19e-4cfd-90ea-dca559866882' 2024-11-20T09:00:58,422 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T09:00:58,422 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "caf50035-b19e-4cfd-90ea-dca559866882" 2024-11-20T09:00:58,422 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4460d48d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T09:00:58,422 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [be1eb2620e0e,44373,-1] 2024-11-20T09:00:58,422 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T09:00:58,423 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T09:00:58,423 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40386, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T09:00:58,424 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6cf81628, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T09:00:58,424 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42125f21, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T09:00:58,424 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request be1eb2620e0e,44373,-1 for getting cluster id 2024-11-20T09:00:58,424 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T09:00:58,425 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'caf50035-b19e-4cfd-90ea-dca559866882' 2024-11-20T09:00:58,425 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T09:00:58,425 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "caf50035-b19e-4cfd-90ea-dca559866882" 2024-11-20T09:00:58,425 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@17676ff2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T09:00:58,425 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [be1eb2620e0e,44373,-1] 2024-11-20T09:00:58,425 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T09:00:58,425 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T09:00:58,426 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40398, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T09:00:58,426 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5e8819a6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T09:00:58,427 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64b126a0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T09:00:58,427 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request be1eb2620e0e,44373,-1 for getting cluster id 2024-11-20T09:00:58,427 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T09:00:58,427 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'caf50035-b19e-4cfd-90ea-dca559866882' 2024-11-20T09:00:58,427 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T09:00:58,427 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "caf50035-b19e-4cfd-90ea-dca559866882" 2024-11-20T09:00:58,427 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2903d7e3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T09:00:58,427 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [be1eb2620e0e,44373,-1] 2024-11-20T09:00:58,428 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T09:00:58,428 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T09:00:58,428 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40426, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T09:00:58,429 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4f215043, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T09:00:58,429 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4fa71035, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T09:00:58,429 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request be1eb2620e0e,44373,-1 for getting cluster id 2024-11-20T09:00:58,429 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T09:00:58,430 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'caf50035-b19e-4cfd-90ea-dca559866882' 2024-11-20T09:00:58,430 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T09:00:58,430 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "caf50035-b19e-4cfd-90ea-dca559866882" 2024-11-20T09:00:58,430 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2465fe11, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T09:00:58,430 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [be1eb2620e0e,44373,-1] 2024-11-20T09:00:58,430 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T09:00:58,431 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T09:00:58,431 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40442, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T09:00:58,432 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f71266d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T09:00:58,432 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5135eed0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T09:00:58,432 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request be1eb2620e0e,44373,-1 for getting cluster id 2024-11-20T09:00:58,432 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T09:00:58,433 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'caf50035-b19e-4cfd-90ea-dca559866882' 2024-11-20T09:00:58,433 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T09:00:58,433 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "caf50035-b19e-4cfd-90ea-dca559866882" 2024-11-20T09:00:58,433 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@461d2f88, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T09:00:58,433 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [be1eb2620e0e,44373,-1] 2024-11-20T09:00:58,433 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T09:00:58,433 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T09:00:58,434 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40458, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T09:00:58,434 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f45392, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T09:00:58,435 DEBUG [Thread-1726 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T09:00:58,435 DEBUG [Thread-1728 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T09:00:58,435 DEBUG [Thread-1722 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T09:00:58,435 DEBUG [Thread-1730 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T09:00:58,435 DEBUG [Thread-1713 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T09:00:58,436 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=be1eb2620e0e,34223,1732093159960, seqNum=-1] 2024-11-20T09:00:58,436 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T09:00:58,436 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=be1eb2620e0e,34223,1732093159960, seqNum=-1] 2024-11-20T09:00:58,436 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=be1eb2620e0e,34223,1732093159960, seqNum=-1] 2024-11-20T09:00:58,436 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=be1eb2620e0e,34223,1732093159960, seqNum=-1] 2024-11-20T09:00:58,436 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T09:00:58,436 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T09:00:58,436 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T09:00:58,436 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T09:00:58,436 DEBUG [Thread-1715 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T09:00:58,437 DEBUG [Thread-1711 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T09:00:58,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44373 {}] procedure2.ProcedureExecutor(1139): Stored pid=93, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees 2024-11-20T09:00:58,437 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54938, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T09:00:58,438 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54922, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T09:00:58,438 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=be1eb2620e0e,34223,1732093159960, seqNum=-1] 2024-11-20T09:00:58,438 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54942, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T09:00:58,438 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=be1eb2620e0e,34223,1732093159960, seqNum=-1] 2024-11-20T09:00:58,438 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54944, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T09:00:58,438 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T09:00:58,438 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T09:00:58,438 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=be1eb2620e0e,34223,1732093159960, seqNum=-1] 2024-11-20T09:00:58,438 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T09:00:58,438 DEBUG [Thread-1717 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T09:00:58,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-20T09:00:58,439 DEBUG [Thread-1719 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T09:00:58,439 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54954, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T09:00:58,439 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=93, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T09:00:58,439 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54966, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T09:00:58,439 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=be1eb2620e0e,34223,1732093159960, seqNum=-1] 2024-11-20T09:00:58,439 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=be1eb2620e0e,34223,1732093159960, seqNum=-1] 2024-11-20T09:00:58,440 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54972, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T09:00:58,440 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T09:00:58,440 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T09:00:58,440 DEBUG [Thread-1724 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T09:00:58,440 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=93, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T09:00:58,440 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=94, ppid=93, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T09:00:58,441 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5] 2024-11-20T09:00:58,441 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54984, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T09:00:58,442 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5] 2024-11-20T09:00:58,442 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54996, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T09:00:58,443 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5] 2024-11-20T09:00:58,443 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=be1eb2620e0e,34223,1732093159960, seqNum=-1] 2024-11-20T09:00:58,443 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T09:00:58,443 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5] 2024-11-20T09:00:58,445 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55002, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T09:00:58,445 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5] 2024-11-20T09:00:58,445 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5] 2024-11-20T09:00:58,446 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5] 2024-11-20T09:00:58,446 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5] 2024-11-20T09:00:58,447 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5] 2024-11-20T09:00:58,448 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5] 2024-11-20T09:00:58,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(8855): Flush requested on b3cbab3048a5449de36008b3c0424a49 2024-11-20T09:00:58,455 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T09:00:58,455 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b3cbab3048a5449de36008b3c0424a49 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-20T09:00:58,490 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120e528d9073a1544bfa1c0a3605a2b601f_b3cbab3048a5449de36008b3c0424a49 is 50, key is test_row_0/A:col10/1732093258454/Put/seqid=0 2024-11-20T09:00:58,497 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742205_1381 (size=17034) 2024-11-20T09:00:58,506 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:58,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54922 deadline: 1732093318503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:58,507 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:58,507 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:58,507 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:58,514 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:58,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54996 deadline: 1732093318504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:58,515 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:58,515 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:58,515 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:58,515 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:58,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54954 deadline: 1732093318505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:58,515 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:58,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54984 deadline: 1732093318505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:58,515 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:58,515 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:58,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54972 deadline: 1732093318506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:58,515 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:58,515 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:58,515 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:58,516 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:58,516 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:58,516 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:58,516 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:58,516 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:58,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-20T09:00:58,595 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-20T09:00:58,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49. 2024-11-20T09:00:58,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49. as already flushing 2024-11-20T09:00:58,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49. 2024-11-20T09:00:58,596 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => b3cbab3048a5449de36008b3c0424a49, NAME => 'TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:58,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => b3cbab3048a5449de36008b3c0424a49, NAME => 'TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:58,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b3cbab3048a5449de36008b3c0424a49, NAME => 'TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b3cbab3048a5449de36008b3c0424a49, NAME => 'TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:58,624 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:58,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54922 deadline: 1732093318617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:58,625 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:58,625 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:58,625 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:58,625 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:58,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54996 deadline: 1732093318617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:58,625 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:58,625 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:58,625 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:58,625 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:58,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54954 deadline: 1732093318617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:58,626 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:58,626 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:58,626 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:58,635 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:58,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54984 deadline: 1732093318626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:58,636 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:58,636 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:58,636 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:58,636 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:58,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54972 deadline: 1732093318627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:58,636 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:58,636 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:58,636 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:58,749 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-20T09:00:58,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49. 2024-11-20T09:00:58,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49. as already flushing 2024-11-20T09:00:58,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49. 2024-11-20T09:00:58,749 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => b3cbab3048a5449de36008b3c0424a49, NAME => 'TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:58,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => b3cbab3048a5449de36008b3c0424a49, NAME => 'TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:58,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b3cbab3048a5449de36008b3c0424a49, NAME => 'TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b3cbab3048a5449de36008b3c0424a49, NAME => 'TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:58,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-20T09:00:58,835 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:58,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54954 deadline: 1732093318827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:58,835 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:58,835 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:58,835 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:58,841 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:58,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54922 deadline: 1732093318836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:58,841 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:58,841 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:58,841 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:58,846 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:58,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54996 deadline: 1732093318837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:58,846 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:58,846 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:58,846 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:58,850 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:58,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54984 deadline: 1732093318847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:58,850 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:58,850 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:58,850 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:58,850 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:58,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54972 deadline: 1732093318847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:58,851 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:58,851 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:58,851 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:58,898 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:00:58,901 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120e528d9073a1544bfa1c0a3605a2b601f_b3cbab3048a5449de36008b3c0424a49 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120e528d9073a1544bfa1c0a3605a2b601f_b3cbab3048a5449de36008b3c0424a49 2024-11-20T09:00:58,902 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-20T09:00:58,902 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/.tmp/A/2968f7d8b4f4474184806fdcfba08b67, store: [table=TestAcidGuarantees family=A region=b3cbab3048a5449de36008b3c0424a49] 2024-11-20T09:00:58,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49. 2024-11-20T09:00:58,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49. as already flushing 2024-11-20T09:00:58,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49. 2024-11-20T09:00:58,902 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => b3cbab3048a5449de36008b3c0424a49, NAME => 'TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:58,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => b3cbab3048a5449de36008b3c0424a49, NAME => 'TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:58,903 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/.tmp/A/2968f7d8b4f4474184806fdcfba08b67 is 175, key is test_row_0/A:col10/1732093258454/Put/seqid=0 2024-11-20T09:00:58,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b3cbab3048a5449de36008b3c0424a49, NAME => 'TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b3cbab3048a5449de36008b3c0424a49, NAME => 'TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:58,906 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742206_1382 (size=48139) 2024-11-20T09:00:59,055 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-20T09:00:59,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49. 2024-11-20T09:00:59,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49. as already flushing 2024-11-20T09:00:59,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49. 2024-11-20T09:00:59,055 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => b3cbab3048a5449de36008b3c0424a49, NAME => 'TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:59,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => b3cbab3048a5449de36008b3c0424a49, NAME => 'TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:59,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b3cbab3048a5449de36008b3c0424a49, NAME => 'TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b3cbab3048a5449de36008b3c0424a49, NAME => 'TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:59,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-20T09:00:59,153 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:59,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54922 deadline: 1732093319147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:59,154 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:59,154 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:59,154 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:59,154 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:59,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54954 deadline: 1732093319147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:59,154 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:59,154 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:59,154 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:59,164 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:59,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54996 deadline: 1732093319156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:59,165 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:59,165 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:59,165 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:59,165 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:59,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54984 deadline: 1732093319156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:59,165 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:59,165 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:59,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54972 deadline: 1732093319157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:59,165 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:59,165 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:59,166 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:59,166 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:59,166 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:59,208 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-20T09:00:59,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49. 2024-11-20T09:00:59,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49. as already flushing 2024-11-20T09:00:59,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49. 2024-11-20T09:00:59,208 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => b3cbab3048a5449de36008b3c0424a49, NAME => 'TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:59,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => b3cbab3048a5449de36008b3c0424a49, NAME => 'TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:59,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b3cbab3048a5449de36008b3c0424a49, NAME => 'TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b3cbab3048a5449de36008b3c0424a49, NAME => 'TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:59,307 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=16, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/.tmp/A/2968f7d8b4f4474184806fdcfba08b67 2024-11-20T09:00:59,331 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/.tmp/B/378d6d0381224951966cef5fef728d42 is 50, key is test_row_0/B:col10/1732093258454/Put/seqid=0 2024-11-20T09:00:59,334 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742207_1383 (size=12001) 2024-11-20T09:00:59,336 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/.tmp/B/378d6d0381224951966cef5fef728d42 2024-11-20T09:00:59,357 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/.tmp/C/2b6dae35ba0c4d1287abf6f57694e7a0 is 50, key is test_row_0/C:col10/1732093258454/Put/seqid=0 2024-11-20T09:00:59,361 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-20T09:00:59,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49. 2024-11-20T09:00:59,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49. as already flushing 2024-11-20T09:00:59,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49. 2024-11-20T09:00:59,361 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => b3cbab3048a5449de36008b3c0424a49, NAME => 'TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:59,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => b3cbab3048a5449de36008b3c0424a49, NAME => 'TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:59,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b3cbab3048a5449de36008b3c0424a49, NAME => 'TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b3cbab3048a5449de36008b3c0424a49, NAME => 'TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:59,366 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742208_1384 (size=12001) 2024-11-20T09:00:59,514 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-20T09:00:59,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49. 2024-11-20T09:00:59,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49. as already flushing 2024-11-20T09:00:59,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49. 2024-11-20T09:00:59,514 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => b3cbab3048a5449de36008b3c0424a49, NAME => 'TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:59,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => b3cbab3048a5449de36008b3c0424a49, NAME => 'TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:59,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b3cbab3048a5449de36008b3c0424a49, NAME => 'TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b3cbab3048a5449de36008b3c0424a49, NAME => 'TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:59,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-20T09:00:59,667 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-20T09:00:59,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49. 2024-11-20T09:00:59,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49. as already flushing 2024-11-20T09:00:59,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49. 2024-11-20T09:00:59,667 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => b3cbab3048a5449de36008b3c0424a49, NAME => 'TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:59,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => b3cbab3048a5449de36008b3c0424a49, NAME => 'TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:59,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b3cbab3048a5449de36008b3c0424a49, NAME => 'TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b3cbab3048a5449de36008b3c0424a49, NAME => 'TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:00:59,675 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:59,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54922 deadline: 1732093319666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:59,676 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:59,676 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:59,676 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:59,676 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:59,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54954 deadline: 1732093319666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:59,676 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:59,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54996 deadline: 1732093319666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:59,676 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:59,676 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:59,676 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:59,676 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:59,676 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:59,676 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:59,681 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:59,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54984 deadline: 1732093319677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:59,681 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:59,681 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:59,681 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:59,681 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:00:59,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54972 deadline: 1732093319677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:00:59,682 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:59,682 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:00:59,682 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:00:59,767 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/.tmp/C/2b6dae35ba0c4d1287abf6f57694e7a0 2024-11-20T09:00:59,771 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/.tmp/A/2968f7d8b4f4474184806fdcfba08b67 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/A/2968f7d8b4f4474184806fdcfba08b67 2024-11-20T09:00:59,775 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/A/2968f7d8b4f4474184806fdcfba08b67, entries=250, sequenceid=16, filesize=47.0 K 2024-11-20T09:00:59,776 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/.tmp/B/378d6d0381224951966cef5fef728d42 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/B/378d6d0381224951966cef5fef728d42 2024-11-20T09:00:59,779 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/B/378d6d0381224951966cef5fef728d42, entries=150, sequenceid=16, filesize=11.7 K 2024-11-20T09:00:59,779 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/.tmp/C/2b6dae35ba0c4d1287abf6f57694e7a0 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/C/2b6dae35ba0c4d1287abf6f57694e7a0 2024-11-20T09:00:59,783 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/C/2b6dae35ba0c4d1287abf6f57694e7a0, entries=150, sequenceid=16, filesize=11.7 K 2024-11-20T09:00:59,784 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=147.60 KB/151140 for b3cbab3048a5449de36008b3c0424a49 in 1328ms, sequenceid=16, compaction requested=false 2024-11-20T09:00:59,784 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b3cbab3048a5449de36008b3c0424a49: 2024-11-20T09:00:59,820 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-20T09:00:59,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49. 2024-11-20T09:00:59,821 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2902): Flushing b3cbab3048a5449de36008b3c0424a49 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-20T09:00:59,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411202302403566f44bb89dfdf15d817e6f83_b3cbab3048a5449de36008b3c0424a49 is 50, key is test_row_0/A:col10/1732093258503/Put/seqid=0 2024-11-20T09:00:59,836 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742209_1385 (size=12154) 2024-11-20T09:01:00,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:00,241 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411202302403566f44bb89dfdf15d817e6f83_b3cbab3048a5449de36008b3c0424a49 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411202302403566f44bb89dfdf15d817e6f83_b3cbab3048a5449de36008b3c0424a49 2024-11-20T09:01:00,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/.tmp/A/c50bbdd21fc24b409352f29ab7e76e0c, store: [table=TestAcidGuarantees family=A region=b3cbab3048a5449de36008b3c0424a49] 2024-11-20T09:01:00,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/.tmp/A/c50bbdd21fc24b409352f29ab7e76e0c is 175, key is test_row_0/A:col10/1732093258503/Put/seqid=0 2024-11-20T09:01:00,246 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742210_1386 (size=30955) 2024-11-20T09:01:00,246 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=41, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/.tmp/A/c50bbdd21fc24b409352f29ab7e76e0c 2024-11-20T09:01:00,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/.tmp/B/3b517e40b081499c82dec600a2230b86 is 50, key is test_row_0/B:col10/1732093258503/Put/seqid=0 2024-11-20T09:01:00,255 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-20T09:01:00,269 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742211_1387 (size=12001) 2024-11-20T09:01:00,270 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=41 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/.tmp/B/3b517e40b081499c82dec600a2230b86 2024-11-20T09:01:00,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/.tmp/C/46a62284bdf545599c658ae540c642f7 is 50, key is test_row_0/C:col10/1732093258503/Put/seqid=0 2024-11-20T09:01:00,287 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742212_1388 (size=12001) 2024-11-20T09:01:00,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-20T09:01:00,689 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=41 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/.tmp/C/46a62284bdf545599c658ae540c642f7 2024-11-20T09:01:00,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/.tmp/A/c50bbdd21fc24b409352f29ab7e76e0c as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/A/c50bbdd21fc24b409352f29ab7e76e0c 2024-11-20T09:01:00,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(8855): Flush requested on b3cbab3048a5449de36008b3c0424a49 2024-11-20T09:01:00,693 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49. as already flushing 2024-11-20T09:01:00,697 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/A/c50bbdd21fc24b409352f29ab7e76e0c, entries=150, sequenceid=41, filesize=30.2 K 2024-11-20T09:01:00,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/.tmp/B/3b517e40b081499c82dec600a2230b86 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/B/3b517e40b081499c82dec600a2230b86 2024-11-20T09:01:00,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:00,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:00,701 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/B/3b517e40b081499c82dec600a2230b86, entries=150, sequenceid=41, filesize=11.7 K 2024-11-20T09:01:00,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:00,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:00,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/.tmp/C/46a62284bdf545599c658ae540c642f7 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/C/46a62284bdf545599c658ae540c642f7 2024-11-20T09:01:00,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:00,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:00,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:00,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:00,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:00,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:00,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:00,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:00,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:00,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:00,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:00,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:00,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:00,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:00,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:00,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:00,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:00,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:00,706 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/C/46a62284bdf545599c658ae540c642f7, entries=150, sequenceid=41, filesize=11.7 K 2024-11-20T09:01:00,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:00,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:00,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:00,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:00,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:00,707 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=40.25 KB/41220 for b3cbab3048a5449de36008b3c0424a49 in 886ms, sequenceid=41, compaction requested=false 2024-11-20T09:01:00,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:00,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2603): Flush status journal for b3cbab3048a5449de36008b3c0424a49: 2024-11-20T09:01:00,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49. 2024-11-20T09:01:00,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=94 2024-11-20T09:01:00,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:00,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:00,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:00,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.HMaster(4169): Remote procedure done, pid=94 2024-11-20T09:01:00,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:00,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:00,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:00,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:00,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:00,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:00,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:00,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:00,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:00,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:00,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:00,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:00,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:00,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:00,711 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=94, resume processing ppid=93 2024-11-20T09:01:00,711 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=94, ppid=93, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2690 sec 2024-11-20T09:01:00,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:00,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:00,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:00,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:00,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:00,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:00,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:00,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:00,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:00,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:00,713 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=93, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees in 2.2750 sec 2024-11-20T09:01:00,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:00,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:00,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:00,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:00,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:00,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:00,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:00,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:00,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:00,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:00,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:00,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:00,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:00,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:00,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:00,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:00,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:00,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:00,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:00,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:00,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:00,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:00,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:00,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:00,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:00,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:00,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:00,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:00,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:00,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:00,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:00,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:00,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:00,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:00,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:00,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:00,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:00,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:00,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:00,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:00,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:00,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:00,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:00,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:00,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:00,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:00,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:00,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:00,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:00,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:00,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:00,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:00,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:00,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:00,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:00,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:00,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:00,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:00,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:00,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:00,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:00,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:00,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:00,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:00,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:00,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:00,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:00,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:00,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:00,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:00,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:00,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:00,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:00,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:00,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:00,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:00,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:00,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:00,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:00,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:00,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:00,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:00,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:00,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:00,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:00,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:00,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:00,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:00,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:00,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:00,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:00,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:00,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:00,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:00,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:00,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:00,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:00,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:00,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:00,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:00,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:00,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(8855): Flush requested on b3cbab3048a5449de36008b3c0424a49 2024-11-20T09:01:00,738 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T09:01:00,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:00,738 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b3cbab3048a5449de36008b3c0424a49 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-20T09:01:00,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:00,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:00,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:00,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:00,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:00,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:00,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:00,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:00,745 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411202561b03189d44ff997bf3c54dd0914d6_b3cbab3048a5449de36008b3c0424a49 is 50, key is test_row_0/A:col10/1732093260736/Put/seqid=0 2024-11-20T09:01:00,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:00,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:00,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:00,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:00,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:00,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:00,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:00,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:00,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:00,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:00,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:00,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:00,753 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742213_1389 (size=17034) 2024-11-20T09:01:00,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:00,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:00,754 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:00,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:00,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:00,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:00,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:00,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:00,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:00,759 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411202561b03189d44ff997bf3c54dd0914d6_b3cbab3048a5449de36008b3c0424a49 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411202561b03189d44ff997bf3c54dd0914d6_b3cbab3048a5449de36008b3c0424a49 2024-11-20T09:01:00,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:00,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:00,760 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/.tmp/A/7deae91156e74fdfb33d0701c934bfa8, store: [table=TestAcidGuarantees family=A region=b3cbab3048a5449de36008b3c0424a49] 2024-11-20T09:01:00,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:00,760 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/.tmp/A/7deae91156e74fdfb33d0701c934bfa8 is 175, key is test_row_0/A:col10/1732093260736/Put/seqid=0 2024-11-20T09:01:00,767 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742214_1390 (size=48139) 2024-11-20T09:01:00,803 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:00,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54996 deadline: 1732093320786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:00,804 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:00,804 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:00,804 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:00,804 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:00,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54922 deadline: 1732093320790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:00,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54984 deadline: 1732093320789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:00,804 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:00,804 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:00,804 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:00,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54972 deadline: 1732093320796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:00,804 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:00,804 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:00,804 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:00,804 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:00,804 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:00,804 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:00,805 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:00,805 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:00,808 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:00,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54954 deadline: 1732093320797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:00,808 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:00,808 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:00,808 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:00,918 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:00,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54996 deadline: 1732093320906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:00,919 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:00,919 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:00,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54922 deadline: 1732093320907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:00,919 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:00,919 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:00,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54984 deadline: 1732093320907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:00,919 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:00,919 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:00,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54972 deadline: 1732093320907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:00,919 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:00,919 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:00,919 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:00,919 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:00,919 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:00,919 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:00,919 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:00,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54954 deadline: 1732093320917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:00,920 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:00,920 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:00,920 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:00,920 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:00,920 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:00,920 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:01,129 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:01,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54996 deadline: 1732093321127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:01,130 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:01,130 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:01,130 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:01,130 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:01,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54922 deadline: 1732093321127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:01,130 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:01,130 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:01,130 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:01,130 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:01,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54984 deadline: 1732093321127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:01,131 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:01,131 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:01,131 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:01,131 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:01,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54972 deadline: 1732093321127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:01,131 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:01,131 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:01,131 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:01,131 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:01,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54954 deadline: 1732093321127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:01,131 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:01,132 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:01,132 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:01,168 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=53, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/.tmp/A/7deae91156e74fdfb33d0701c934bfa8 2024-11-20T09:01:01,175 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/.tmp/B/2854423aa2e04ff29efd79196b434b5d is 50, key is test_row_0/B:col10/1732093260736/Put/seqid=0 2024-11-20T09:01:01,180 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742215_1391 (size=12001) 2024-11-20T09:01:01,447 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:01,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54996 deadline: 1732093321437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:01,447 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:01,447 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:01,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54984 deadline: 1732093321437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:01,447 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:01,447 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:01,447 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:01,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54922 deadline: 1732093321437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:01,447 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:01,448 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:01,448 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:01,448 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:01,448 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:01,448 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:01,448 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:01,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54954 deadline: 1732093321438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:01,448 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:01,448 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:01,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54972 deadline: 1732093321438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:01,448 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:01,448 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:01,448 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:01,448 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:01,448 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:01,581 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=53 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/.tmp/B/2854423aa2e04ff29efd79196b434b5d 2024-11-20T09:01:01,588 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/.tmp/C/6abfa7120f714f8cb577190349335860 is 50, key is test_row_0/C:col10/1732093260736/Put/seqid=0 2024-11-20T09:01:01,591 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742216_1392 (size=12001) 2024-11-20T09:01:01,957 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:01,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54996 deadline: 1732093321956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:01,957 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:01,957 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:01,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54922 deadline: 1732093321957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:01,957 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:01,957 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:01,958 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:01,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54984 deadline: 1732093321957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:01,958 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:01,958 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:01,958 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:01,958 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:01,958 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:01,958 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:01,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54954 deadline: 1732093321957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:01,958 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:01,958 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:01,958 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:01,958 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:01,958 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:01,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54972 deadline: 1732093321957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:01,959 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:01,959 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:01,959 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:01,992 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=53 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/.tmp/C/6abfa7120f714f8cb577190349335860 2024-11-20T09:01:01,996 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/.tmp/A/7deae91156e74fdfb33d0701c934bfa8 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/A/7deae91156e74fdfb33d0701c934bfa8 2024-11-20T09:01:02,000 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/A/7deae91156e74fdfb33d0701c934bfa8, entries=250, sequenceid=53, filesize=47.0 K 2024-11-20T09:01:02,001 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/.tmp/B/2854423aa2e04ff29efd79196b434b5d as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/B/2854423aa2e04ff29efd79196b434b5d 2024-11-20T09:01:02,004 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/B/2854423aa2e04ff29efd79196b434b5d, entries=150, sequenceid=53, filesize=11.7 K 2024-11-20T09:01:02,005 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/.tmp/C/6abfa7120f714f8cb577190349335860 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/C/6abfa7120f714f8cb577190349335860 2024-11-20T09:01:02,008 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/C/6abfa7120f714f8cb577190349335860, entries=150, sequenceid=53, filesize=11.7 K 2024-11-20T09:01:02,009 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for b3cbab3048a5449de36008b3c0424a49 in 1271ms, sequenceid=53, compaction requested=true 2024-11-20T09:01:02,009 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b3cbab3048a5449de36008b3c0424a49: 2024-11-20T09:01:02,010 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b3cbab3048a5449de36008b3c0424a49:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T09:01:02,010 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T09:01:02,010 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b3cbab3048a5449de36008b3c0424a49:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T09:01:02,010 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T09:01:02,010 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T09:01:02,010 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b3cbab3048a5449de36008b3c0424a49:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T09:01:02,010 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T09:01:02,010 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T09:01:02,011 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T09:01:02,011 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 127233 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T09:01:02,011 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HStore(1541): b3cbab3048a5449de36008b3c0424a49/A is initiating minor compaction (all files) 2024-11-20T09:01:02,011 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1541): b3cbab3048a5449de36008b3c0424a49/B is initiating minor compaction (all files) 2024-11-20T09:01:02,011 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b3cbab3048a5449de36008b3c0424a49/A in TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49. 2024-11-20T09:01:02,011 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b3cbab3048a5449de36008b3c0424a49/B in TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49. 2024-11-20T09:01:02,011 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/A/2968f7d8b4f4474184806fdcfba08b67, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/A/c50bbdd21fc24b409352f29ab7e76e0c, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/A/7deae91156e74fdfb33d0701c934bfa8] into tmpdir=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/.tmp, totalSize=124.3 K 2024-11-20T09:01:02,011 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/B/378d6d0381224951966cef5fef728d42, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/B/3b517e40b081499c82dec600a2230b86, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/B/2854423aa2e04ff29efd79196b434b5d] into tmpdir=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/.tmp, totalSize=35.2 K 2024-11-20T09:01:02,011 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49. 2024-11-20T09:01:02,011 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49. files: [hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/A/2968f7d8b4f4474184806fdcfba08b67, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/A/c50bbdd21fc24b409352f29ab7e76e0c, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/A/7deae91156e74fdfb33d0701c934bfa8] 2024-11-20T09:01:02,012 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting 378d6d0381224951966cef5fef728d42, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1732093258449 2024-11-20T09:01:02,012 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2968f7d8b4f4474184806fdcfba08b67, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1732093258446 2024-11-20T09:01:02,012 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.Compactor(225): Compacting c50bbdd21fc24b409352f29ab7e76e0c, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1732093258503 2024-11-20T09:01:02,012 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting 3b517e40b081499c82dec600a2230b86, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1732093258503 2024-11-20T09:01:02,012 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7deae91156e74fdfb33d0701c934bfa8, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1732093260695 2024-11-20T09:01:02,012 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting 2854423aa2e04ff29efd79196b434b5d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1732093260695 2024-11-20T09:01:02,019 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=b3cbab3048a5449de36008b3c0424a49] 2024-11-20T09:01:02,020 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b3cbab3048a5449de36008b3c0424a49#B#compaction#335 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-20T09:01:02,020 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/.tmp/B/bf51b2dafc2949489b704d0fd194ca28 is 50, key is test_row_0/B:col10/1732093260736/Put/seqid=0 2024-11-20T09:01:02,021 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241120c64314c6a4584cc09383f4c73f84ca69_b3cbab3048a5449de36008b3c0424a49 store=[table=TestAcidGuarantees family=A region=b3cbab3048a5449de36008b3c0424a49] 2024-11-20T09:01:02,022 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241120c64314c6a4584cc09383f4c73f84ca69_b3cbab3048a5449de36008b3c0424a49, store=[table=TestAcidGuarantees family=A region=b3cbab3048a5449de36008b3c0424a49] 2024-11-20T09:01:02,022 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120c64314c6a4584cc09383f4c73f84ca69_b3cbab3048a5449de36008b3c0424a49 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=b3cbab3048a5449de36008b3c0424a49] 2024-11-20T09:01:02,031 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742217_1393 (size=4469) 2024-11-20T09:01:02,036 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742218_1394 (size=12104) 2024-11-20T09:01:02,041 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/.tmp/B/bf51b2dafc2949489b704d0fd194ca28 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/B/bf51b2dafc2949489b704d0fd194ca28 2024-11-20T09:01:02,047 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b3cbab3048a5449de36008b3c0424a49/B of b3cbab3048a5449de36008b3c0424a49 into bf51b2dafc2949489b704d0fd194ca28(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T09:01:02,047 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b3cbab3048a5449de36008b3c0424a49: 2024-11-20T09:01:02,047 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., storeName=b3cbab3048a5449de36008b3c0424a49/B, priority=13, startTime=1732093262010; duration=0sec 2024-11-20T09:01:02,047 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T09:01:02,047 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b3cbab3048a5449de36008b3c0424a49:B 2024-11-20T09:01:02,047 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T09:01:02,048 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T09:01:02,048 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1541): b3cbab3048a5449de36008b3c0424a49/C is initiating minor compaction (all files) 2024-11-20T09:01:02,048 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b3cbab3048a5449de36008b3c0424a49/C in TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49. 2024-11-20T09:01:02,048 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/C/2b6dae35ba0c4d1287abf6f57694e7a0, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/C/46a62284bdf545599c658ae540c642f7, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/C/6abfa7120f714f8cb577190349335860] into tmpdir=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/.tmp, totalSize=35.2 K 2024-11-20T09:01:02,048 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting 2b6dae35ba0c4d1287abf6f57694e7a0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1732093258449 2024-11-20T09:01:02,048 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting 46a62284bdf545599c658ae540c642f7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1732093258503 2024-11-20T09:01:02,049 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting 6abfa7120f714f8cb577190349335860, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1732093260695 2024-11-20T09:01:02,056 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b3cbab3048a5449de36008b3c0424a49#C#compaction#337 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-20T09:01:02,056 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/.tmp/C/87f496937dcc4a2eb1092b30b9a53181 is 50, key is test_row_0/C:col10/1732093260736/Put/seqid=0 2024-11-20T09:01:02,064 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742219_1395 (size=12104) 2024-11-20T09:01:02,068 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/.tmp/C/87f496937dcc4a2eb1092b30b9a53181 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/C/87f496937dcc4a2eb1092b30b9a53181 2024-11-20T09:01:02,072 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b3cbab3048a5449de36008b3c0424a49/C of b3cbab3048a5449de36008b3c0424a49 into 87f496937dcc4a2eb1092b30b9a53181(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T09:01:02,072 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b3cbab3048a5449de36008b3c0424a49: 2024-11-20T09:01:02,072 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., storeName=b3cbab3048a5449de36008b3c0424a49/C, priority=13, startTime=1732093262010; duration=0sec 2024-11-20T09:01:02,072 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T09:01:02,072 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b3cbab3048a5449de36008b3c0424a49:C 2024-11-20T09:01:02,432 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b3cbab3048a5449de36008b3c0424a49#A#compaction#336 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T09:01:02,433 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/.tmp/A/f244cb3d6ced47fb9ce45d828f7776b1 is 175, key is test_row_0/A:col10/1732093260736/Put/seqid=0 2024-11-20T09:01:02,437 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742220_1396 (size=31058) 2024-11-20T09:01:02,442 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/.tmp/A/f244cb3d6ced47fb9ce45d828f7776b1 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/A/f244cb3d6ced47fb9ce45d828f7776b1 2024-11-20T09:01:02,449 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b3cbab3048a5449de36008b3c0424a49/A of b3cbab3048a5449de36008b3c0424a49 into f244cb3d6ced47fb9ce45d828f7776b1(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T09:01:02,450 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b3cbab3048a5449de36008b3c0424a49: 2024-11-20T09:01:02,450 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., storeName=b3cbab3048a5449de36008b3c0424a49/A, priority=13, startTime=1732093262009; duration=0sec 2024-11-20T09:01:02,450 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T09:01:02,450 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b3cbab3048a5449de36008b3c0424a49:A 2024-11-20T09:01:02,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-20T09:01:02,597 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T09:01:02,598 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T09:01:02,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] procedure2.ProcedureExecutor(1139): Stored pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees 2024-11-20T09:01:02,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-20T09:01:02,600 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T09:01:02,600 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T09:01:02,601 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=96, ppid=95, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T09:01:02,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-20T09:01:02,753 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-20T09:01:02,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49. 2024-11-20T09:01:02,753 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2902): Flushing b3cbab3048a5449de36008b3c0424a49 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-20T09:01:02,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120bf95ce65d3ed45bab803cbffaa632f39_b3cbab3048a5449de36008b3c0424a49 is 50, key is test_row_0/A:col10/1732093260796/Put/seqid=0 2024-11-20T09:01:02,763 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742221_1397 (size=12154) 2024-11-20T09:01:02,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-20T09:01:02,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(8855): Flush requested on b3cbab3048a5449de36008b3c0424a49 2024-11-20T09:01:02,971 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49. as already flushing 2024-11-20T09:01:02,986 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:02,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54922 deadline: 1732093322982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:02,987 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:02,987 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:02,987 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:02,991 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:02,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54984 deadline: 1732093322984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:02,991 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:02,991 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:02,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54954 deadline: 1732093322984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:02,991 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:02,991 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:02,992 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:02,992 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:02,992 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:02,997 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:02,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54972 deadline: 1732093322985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:02,998 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:02,998 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:02,998 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:02,998 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:02,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54996 deadline: 1732093322986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:02,998 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:02,998 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:02,998 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:03,100 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:03,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54922 deadline: 1732093323097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:03,101 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:03,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54984 deadline: 1732093323097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:03,101 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:03,101 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:03,101 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:03,101 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:03,101 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:03,101 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:03,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54954 deadline: 1732093323097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:03,101 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:03,101 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:03,101 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:03,101 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:03,111 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:03,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54972 deadline: 1732093323107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:03,111 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:03,111 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:03,111 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:03,111 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:03,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54996 deadline: 1732093323107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:03,112 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:03,112 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:03,112 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:03,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:03,168 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120bf95ce65d3ed45bab803cbffaa632f39_b3cbab3048a5449de36008b3c0424a49 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120bf95ce65d3ed45bab803cbffaa632f39_b3cbab3048a5449de36008b3c0424a49 2024-11-20T09:01:03,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/.tmp/A/ca4d80643bcb432c86c3386b8e625bb1, store: [table=TestAcidGuarantees family=A region=b3cbab3048a5449de36008b3c0424a49] 2024-11-20T09:01:03,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/.tmp/A/ca4d80643bcb432c86c3386b8e625bb1 is 175, key is test_row_0/A:col10/1732093260796/Put/seqid=0 2024-11-20T09:01:03,173 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742222_1398 (size=30955) 2024-11-20T09:01:03,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-20T09:01:03,310 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:03,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54922 deadline: 1732093323307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:03,311 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:03,311 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:03,311 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:03,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54984 deadline: 1732093323307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:03,311 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:03,311 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:03,311 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:03,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54954 deadline: 1732093323307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:03,311 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:03,311 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:03,311 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:03,311 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:03,312 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:03,321 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:03,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54972 deadline: 1732093323317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:03,322 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:03,322 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:03,322 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:03,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54996 deadline: 1732093323317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:03,322 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:03,322 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:03,322 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:03,322 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:03,573 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=80, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/.tmp/A/ca4d80643bcb432c86c3386b8e625bb1 2024-11-20T09:01:03,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/.tmp/B/1a4d327de879499d9c1cc0fc29d3901f is 50, key is test_row_0/B:col10/1732093260796/Put/seqid=0 2024-11-20T09:01:03,585 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742223_1399 (size=12001) 2024-11-20T09:01:03,586 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=80 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/.tmp/B/1a4d327de879499d9c1cc0fc29d3901f 2024-11-20T09:01:03,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/.tmp/C/b00cef00e7f14b98a2fcf415f687f0a3 is 50, key is test_row_0/C:col10/1732093260796/Put/seqid=0 2024-11-20T09:01:03,596 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742224_1400 (size=12001) 2024-11-20T09:01:03,597 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=80 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/.tmp/C/b00cef00e7f14b98a2fcf415f687f0a3 2024-11-20T09:01:03,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/.tmp/A/ca4d80643bcb432c86c3386b8e625bb1 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/A/ca4d80643bcb432c86c3386b8e625bb1 2024-11-20T09:01:03,605 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/A/ca4d80643bcb432c86c3386b8e625bb1, entries=150, sequenceid=80, filesize=30.2 K 2024-11-20T09:01:03,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/.tmp/B/1a4d327de879499d9c1cc0fc29d3901f as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/B/1a4d327de879499d9c1cc0fc29d3901f 2024-11-20T09:01:03,609 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/B/1a4d327de879499d9c1cc0fc29d3901f, entries=150, sequenceid=80, filesize=11.7 K 2024-11-20T09:01:03,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/.tmp/C/b00cef00e7f14b98a2fcf415f687f0a3 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/C/b00cef00e7f14b98a2fcf415f687f0a3 2024-11-20T09:01:03,615 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/C/b00cef00e7f14b98a2fcf415f687f0a3, entries=150, sequenceid=80, filesize=11.7 K 2024-11-20T09:01:03,616 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for b3cbab3048a5449de36008b3c0424a49 in 863ms, sequenceid=80, compaction requested=false 2024-11-20T09:01:03,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2603): Flush status journal for b3cbab3048a5449de36008b3c0424a49: 2024-11-20T09:01:03,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49. 2024-11-20T09:01:03,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=96 2024-11-20T09:01:03,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.HMaster(4169): Remote procedure done, pid=96 2024-11-20T09:01:03,621 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=96, resume processing ppid=95 2024-11-20T09:01:03,621 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=96, ppid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0180 sec 2024-11-20T09:01:03,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(8855): Flush requested on b3cbab3048a5449de36008b3c0424a49 2024-11-20T09:01:03,621 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T09:01:03,622 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b3cbab3048a5449de36008b3c0424a49 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-20T09:01:03,624 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees in 1.0240 sec 2024-11-20T09:01:03,631 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120fcb3f423fb514bf1b1c173378997044d_b3cbab3048a5449de36008b3c0424a49 is 50, key is test_row_0/A:col10/1732093262982/Put/seqid=0 2024-11-20T09:01:03,642 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742225_1401 (size=14594) 2024-11-20T09:01:03,643 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:03,648 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120fcb3f423fb514bf1b1c173378997044d_b3cbab3048a5449de36008b3c0424a49 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120fcb3f423fb514bf1b1c173378997044d_b3cbab3048a5449de36008b3c0424a49 2024-11-20T09:01:03,649 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/.tmp/A/f9dda23b19b74fe6b72fe6a20f2435a6, store: [table=TestAcidGuarantees family=A region=b3cbab3048a5449de36008b3c0424a49] 2024-11-20T09:01:03,650 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/.tmp/A/f9dda23b19b74fe6b72fe6a20f2435a6 is 175, key is test_row_0/A:col10/1732093262982/Put/seqid=0 2024-11-20T09:01:03,655 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742226_1402 (size=39549) 2024-11-20T09:01:03,671 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:03,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54954 deadline: 1732093323660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:03,671 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:03,671 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:03,671 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:03,671 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:03,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54972 deadline: 1732093323661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:03,672 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:03,672 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:03,672 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:03,672 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:03,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54996 deadline: 1732093323667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:03,672 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:03,672 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:03,672 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:03,682 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:03,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54984 deadline: 1732093323668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:03,683 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:03,683 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:03,683 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:03,683 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:03,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54922 deadline: 1732093323669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:03,683 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:03,683 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:03,683 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:03,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-20T09:01:03,737 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T09:01:03,738 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T09:01:03,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] procedure2.ProcedureExecutor(1139): Stored pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees 2024-11-20T09:01:03,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-20T09:01:03,739 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T09:01:03,740 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T09:01:03,740 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=98, ppid=97, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T09:01:03,784 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:03,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54954 deadline: 1732093323776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:03,785 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:03,785 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:03,785 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:03,785 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:03,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54972 deadline: 1732093323777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:03,785 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:03,785 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:03,785 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:03,785 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:03,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54996 deadline: 1732093323777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:03,786 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:03,786 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:03,786 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:03,798 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:03,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54984 deadline: 1732093323787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:03,798 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:03,798 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:03,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54922 deadline: 1732093323787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:03,798 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:03,798 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:03,798 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:03,798 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:03,798 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:03,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-20T09:01:03,893 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-20T09:01:03,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49. 2024-11-20T09:01:03,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49. as already flushing 2024-11-20T09:01:03,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49. 2024-11-20T09:01:03,893 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => b3cbab3048a5449de36008b3c0424a49, NAME => 'TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:03,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => b3cbab3048a5449de36008b3c0424a49, NAME => 'TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:03,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b3cbab3048a5449de36008b3c0424a49, NAME => 'TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b3cbab3048a5449de36008b3c0424a49, NAME => 'TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:03,996 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:03,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54996 deadline: 1732093323986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:03,996 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:03,996 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:03,996 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:04,003 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:04,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54954 deadline: 1732093323997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:04,003 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:04,003 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:04,003 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:04,003 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:04,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54972 deadline: 1732093323997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:04,003 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:04,004 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:04,004 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:04,016 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:04,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54984 deadline: 1732093324007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:04,016 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:04,016 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:04,016 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:04,016 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:04,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54922 deadline: 1732093324007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:04,016 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:04,016 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:04,016 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:04,046 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-20T09:01:04,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49. 2024-11-20T09:01:04,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49. as already flushing 2024-11-20T09:01:04,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49. 2024-11-20T09:01:04,046 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => b3cbab3048a5449de36008b3c0424a49, NAME => 'TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:04,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => b3cbab3048a5449de36008b3c0424a49, NAME => 'TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:04,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b3cbab3048a5449de36008b3c0424a49, NAME => 'TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b3cbab3048a5449de36008b3c0424a49, NAME => 'TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:04,056 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=93, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/.tmp/A/f9dda23b19b74fe6b72fe6a20f2435a6 2024-11-20T09:01:04,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-20T09:01:04,062 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/.tmp/B/f00794dbca9f489494ee0e3611a486c3 is 50, key is test_row_0/B:col10/1732093262982/Put/seqid=0 2024-11-20T09:01:04,065 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742227_1403 (size=12001) 2024-11-20T09:01:04,066 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/.tmp/B/f00794dbca9f489494ee0e3611a486c3 2024-11-20T09:01:04,072 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/.tmp/C/c4541687e2d14c8cb6f018ffe6240ac2 is 50, key is test_row_0/C:col10/1732093262982/Put/seqid=0 2024-11-20T09:01:04,076 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742228_1404 (size=12001) 2024-11-20T09:01:04,199 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-20T09:01:04,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49. 2024-11-20T09:01:04,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49. as already flushing 2024-11-20T09:01:04,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49. 2024-11-20T09:01:04,199 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => b3cbab3048a5449de36008b3c0424a49, NAME => 'TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:04,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => b3cbab3048a5449de36008b3c0424a49, NAME => 'TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:04,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b3cbab3048a5449de36008b3c0424a49, NAME => 'TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b3cbab3048a5449de36008b3c0424a49, NAME => 'TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:04,315 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:04,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54996 deadline: 1732093324306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:04,316 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:04,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54972 deadline: 1732093324307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:04,316 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:04,316 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:04,316 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:04,316 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:04,316 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:04,316 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:04,322 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:04,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54954 deadline: 1732093324317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:04,323 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:04,323 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:04,323 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:04,335 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:04,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54984 deadline: 1732093324327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:04,336 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:04,336 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:04,336 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:04,336 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:04,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54922 deadline: 1732093324327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:04,336 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:04,336 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:04,336 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:04,352 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-20T09:01:04,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49. 2024-11-20T09:01:04,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49. as already flushing 2024-11-20T09:01:04,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49. 2024-11-20T09:01:04,352 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => b3cbab3048a5449de36008b3c0424a49, NAME => 'TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:04,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => b3cbab3048a5449de36008b3c0424a49, NAME => 'TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:04,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b3cbab3048a5449de36008b3c0424a49, NAME => 'TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b3cbab3048a5449de36008b3c0424a49, NAME => 'TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:04,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-20T09:01:04,477 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/.tmp/C/c4541687e2d14c8cb6f018ffe6240ac2 2024-11-20T09:01:04,480 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/.tmp/A/f9dda23b19b74fe6b72fe6a20f2435a6 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/A/f9dda23b19b74fe6b72fe6a20f2435a6 2024-11-20T09:01:04,484 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/A/f9dda23b19b74fe6b72fe6a20f2435a6, entries=200, sequenceid=93, filesize=38.6 K 2024-11-20T09:01:04,485 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/.tmp/B/f00794dbca9f489494ee0e3611a486c3 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/B/f00794dbca9f489494ee0e3611a486c3 2024-11-20T09:01:04,488 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/B/f00794dbca9f489494ee0e3611a486c3, entries=150, sequenceid=93, filesize=11.7 K 2024-11-20T09:01:04,489 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/.tmp/C/c4541687e2d14c8cb6f018ffe6240ac2 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/C/c4541687e2d14c8cb6f018ffe6240ac2 2024-11-20T09:01:04,492 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/C/c4541687e2d14c8cb6f018ffe6240ac2, entries=150, sequenceid=93, filesize=11.7 K 2024-11-20T09:01:04,493 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for b3cbab3048a5449de36008b3c0424a49 in 872ms, sequenceid=93, compaction requested=true 2024-11-20T09:01:04,493 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b3cbab3048a5449de36008b3c0424a49: 2024-11-20T09:01:04,493 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b3cbab3048a5449de36008b3c0424a49:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T09:01:04,493 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T09:01:04,493 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b3cbab3048a5449de36008b3c0424a49:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T09:01:04,493 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T09:01:04,493 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T09:01:04,493 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b3cbab3048a5449de36008b3c0424a49:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T09:01:04,493 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T09:01:04,493 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T09:01:04,494 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101562 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T09:01:04,494 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HStore(1541): b3cbab3048a5449de36008b3c0424a49/A is initiating minor compaction (all files) 2024-11-20T09:01:04,494 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b3cbab3048a5449de36008b3c0424a49/A in TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49. 2024-11-20T09:01:04,495 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/A/f244cb3d6ced47fb9ce45d828f7776b1, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/A/ca4d80643bcb432c86c3386b8e625bb1, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/A/f9dda23b19b74fe6b72fe6a20f2435a6] into tmpdir=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/.tmp, totalSize=99.2 K 2024-11-20T09:01:04,495 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49. 2024-11-20T09:01:04,495 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49. files: [hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/A/f244cb3d6ced47fb9ce45d828f7776b1, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/A/ca4d80643bcb432c86c3386b8e625bb1, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/A/f9dda23b19b74fe6b72fe6a20f2435a6] 2024-11-20T09:01:04,495 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T09:01:04,495 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.Compactor(225): Compacting f244cb3d6ced47fb9ce45d828f7776b1, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1732093260695 2024-11-20T09:01:04,495 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1541): b3cbab3048a5449de36008b3c0424a49/B is initiating minor compaction (all files) 2024-11-20T09:01:04,495 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b3cbab3048a5449de36008b3c0424a49/B in TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49. 2024-11-20T09:01:04,495 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/B/bf51b2dafc2949489b704d0fd194ca28, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/B/1a4d327de879499d9c1cc0fc29d3901f, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/B/f00794dbca9f489494ee0e3611a486c3] into tmpdir=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/.tmp, totalSize=35.3 K 2024-11-20T09:01:04,495 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.Compactor(225): Compacting ca4d80643bcb432c86c3386b8e625bb1, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1732093260786 2024-11-20T09:01:04,496 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting bf51b2dafc2949489b704d0fd194ca28, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1732093260695 2024-11-20T09:01:04,496 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.Compactor(225): Compacting f9dda23b19b74fe6b72fe6a20f2435a6, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1732093262982 2024-11-20T09:01:04,496 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting 1a4d327de879499d9c1cc0fc29d3901f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1732093260786 2024-11-20T09:01:04,496 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting f00794dbca9f489494ee0e3611a486c3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1732093262982 2024-11-20T09:01:04,501 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=b3cbab3048a5449de36008b3c0424a49] 2024-11-20T09:01:04,502 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b3cbab3048a5449de36008b3c0424a49#B#compaction#345 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-20T09:01:04,503 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/.tmp/B/c1b661f8f45c4954ba501cb303767e92 is 50, key is test_row_0/B:col10/1732093262982/Put/seqid=0 2024-11-20T09:01:04,504 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024112055275a3ba96846429f9658a892d04556_b3cbab3048a5449de36008b3c0424a49 store=[table=TestAcidGuarantees family=A region=b3cbab3048a5449de36008b3c0424a49] 2024-11-20T09:01:04,505 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-20T09:01:04,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49. 2024-11-20T09:01:04,505 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2902): Flushing b3cbab3048a5449de36008b3c0424a49 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-20T09:01:04,506 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024112055275a3ba96846429f9658a892d04556_b3cbab3048a5449de36008b3c0424a49, store=[table=TestAcidGuarantees family=A region=b3cbab3048a5449de36008b3c0424a49] 2024-11-20T09:01:04,506 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112055275a3ba96846429f9658a892d04556_b3cbab3048a5449de36008b3c0424a49 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=b3cbab3048a5449de36008b3c0424a49] 2024-11-20T09:01:04,513 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742230_1406 (size=4469) 2024-11-20T09:01:04,514 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742229_1405 (size=12207) 2024-11-20T09:01:04,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120aa871f8628e54960bb8f58ded37f9579_b3cbab3048a5449de36008b3c0424a49 is 50, key is test_row_0/A:col10/1732093263667/Put/seqid=0 2024-11-20T09:01:04,520 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/.tmp/B/c1b661f8f45c4954ba501cb303767e92 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/B/c1b661f8f45c4954ba501cb303767e92 2024-11-20T09:01:04,521 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742231_1407 (size=12154) 2024-11-20T09:01:04,525 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b3cbab3048a5449de36008b3c0424a49/B of b3cbab3048a5449de36008b3c0424a49 into c1b661f8f45c4954ba501cb303767e92(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T09:01:04,525 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b3cbab3048a5449de36008b3c0424a49: 2024-11-20T09:01:04,525 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., storeName=b3cbab3048a5449de36008b3c0424a49/B, priority=13, startTime=1732093264493; duration=0sec 2024-11-20T09:01:04,525 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T09:01:04,525 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b3cbab3048a5449de36008b3c0424a49:B 2024-11-20T09:01:04,525 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T09:01:04,526 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T09:01:04,528 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1541): b3cbab3048a5449de36008b3c0424a49/C is initiating minor compaction (all files) 2024-11-20T09:01:04,528 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b3cbab3048a5449de36008b3c0424a49/C in TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49. 2024-11-20T09:01:04,528 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/C/87f496937dcc4a2eb1092b30b9a53181, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/C/b00cef00e7f14b98a2fcf415f687f0a3, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/C/c4541687e2d14c8cb6f018ffe6240ac2] into tmpdir=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/.tmp, totalSize=35.3 K 2024-11-20T09:01:04,528 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting 87f496937dcc4a2eb1092b30b9a53181, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1732093260695 2024-11-20T09:01:04,529 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting b00cef00e7f14b98a2fcf415f687f0a3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1732093260786 2024-11-20T09:01:04,529 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting c4541687e2d14c8cb6f018ffe6240ac2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1732093262982 2024-11-20T09:01:04,536 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b3cbab3048a5449de36008b3c0424a49#C#compaction#347 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-20T09:01:04,537 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/.tmp/C/65f9289a98334644ae0ca1227626edc6 is 50, key is test_row_0/C:col10/1732093262982/Put/seqid=0 2024-11-20T09:01:04,540 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742232_1408 (size=12207) 2024-11-20T09:01:04,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(8855): Flush requested on b3cbab3048a5449de36008b3c0424a49 2024-11-20T09:01:04,834 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49. as already flushing 2024-11-20T09:01:04,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-20T09:01:04,897 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:04,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54984 deadline: 1732093324878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:04,897 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:04,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54996 deadline: 1732093324878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:04,897 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:04,897 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:04,897 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:04,897 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:04,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54972 deadline: 1732093324878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:04,898 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:04,898 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:04,898 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:04,898 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:04,898 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:04,898 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:04,898 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:04,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54922 deadline: 1732093324879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:04,898 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:04,898 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:04,898 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:04,898 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:04,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54954 deadline: 1732093324879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:04,898 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:04,898 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:04,898 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:04,915 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b3cbab3048a5449de36008b3c0424a49#A#compaction#344 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T09:01:04,915 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/.tmp/A/750522ba63754a5c80d7ce9ed131873f is 175, key is test_row_0/A:col10/1732093262982/Put/seqid=0 2024-11-20T09:01:04,919 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742233_1409 (size=31161) 2024-11-20T09:01:04,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:04,925 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120aa871f8628e54960bb8f58ded37f9579_b3cbab3048a5449de36008b3c0424a49 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120aa871f8628e54960bb8f58ded37f9579_b3cbab3048a5449de36008b3c0424a49 2024-11-20T09:01:04,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/.tmp/A/3ec6a601a47844448516b3baef042ca9, store: [table=TestAcidGuarantees family=A region=b3cbab3048a5449de36008b3c0424a49] 2024-11-20T09:01:04,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/.tmp/A/3ec6a601a47844448516b3baef042ca9 is 175, key is test_row_0/A:col10/1732093263667/Put/seqid=0 2024-11-20T09:01:04,934 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742234_1410 (size=30955) 2024-11-20T09:01:04,944 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/.tmp/C/65f9289a98334644ae0ca1227626edc6 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/C/65f9289a98334644ae0ca1227626edc6 2024-11-20T09:01:04,951 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b3cbab3048a5449de36008b3c0424a49/C of b3cbab3048a5449de36008b3c0424a49 into 65f9289a98334644ae0ca1227626edc6(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T09:01:04,951 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b3cbab3048a5449de36008b3c0424a49: 2024-11-20T09:01:04,951 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., storeName=b3cbab3048a5449de36008b3c0424a49/C, priority=13, startTime=1732093264493; duration=0sec 2024-11-20T09:01:04,951 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T09:01:04,951 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b3cbab3048a5449de36008b3c0424a49:C 2024-11-20T09:01:05,017 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:05,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54984 deadline: 1732093325007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:05,017 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:05,017 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:05,017 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:05,017 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:05,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54996 deadline: 1732093325007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:05,018 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:05,018 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:05,018 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:05,018 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:05,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54972 deadline: 1732093325007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:05,018 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:05,018 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:05,018 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:05,018 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:05,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54922 deadline: 1732093325008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:05,018 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:05,018 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:05,018 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:05,018 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:05,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54954 deadline: 1732093325008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:05,019 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:05,019 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:05,019 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:05,236 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:05,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54984 deadline: 1732093325226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:05,237 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:05,237 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:05,237 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:05,237 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:05,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54996 deadline: 1732093325226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:05,237 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:05,237 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:05,237 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:05,237 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:05,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54972 deadline: 1732093325226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:05,237 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:05,237 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:05,237 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:05,237 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:05,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54922 deadline: 1732093325227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:05,238 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:05,238 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:05,238 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:05,238 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:05,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54954 deadline: 1732093325227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:05,238 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:05,238 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:05,238 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:05,323 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/.tmp/A/750522ba63754a5c80d7ce9ed131873f as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/A/750522ba63754a5c80d7ce9ed131873f 2024-11-20T09:01:05,327 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b3cbab3048a5449de36008b3c0424a49/A of b3cbab3048a5449de36008b3c0424a49 into 750522ba63754a5c80d7ce9ed131873f(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T09:01:05,327 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b3cbab3048a5449de36008b3c0424a49: 2024-11-20T09:01:05,328 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., storeName=b3cbab3048a5449de36008b3c0424a49/A, priority=13, startTime=1732093264493; duration=0sec 2024-11-20T09:01:05,328 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T09:01:05,328 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b3cbab3048a5449de36008b3c0424a49:A 2024-11-20T09:01:05,334 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=116, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/.tmp/A/3ec6a601a47844448516b3baef042ca9 2024-11-20T09:01:05,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/.tmp/B/9e950dc9d2e4492db1716b54755bdced is 50, key is test_row_0/B:col10/1732093263667/Put/seqid=0 2024-11-20T09:01:05,347 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742235_1411 (size=12001) 2024-11-20T09:01:05,559 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:05,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54984 deadline: 1732093325547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:05,559 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:05,559 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:05,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54972 deadline: 1732093325547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:05,559 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:05,560 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:05,560 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:05,560 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:05,560 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:05,560 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:05,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54954 deadline: 1732093325547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:05,560 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:05,560 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:05,560 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:05,560 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:05,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54996 deadline: 1732093325547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:05,560 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:05,561 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:05,560 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:05,561 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:05,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54922 deadline: 1732093325547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:05,561 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:05,561 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:05,561 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:05,747 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/.tmp/B/9e950dc9d2e4492db1716b54755bdced 2024-11-20T09:01:05,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/.tmp/C/6bb91811e3a84dad8ac240d26209cec0 is 50, key is test_row_0/C:col10/1732093263667/Put/seqid=0 2024-11-20T09:01:05,758 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742236_1412 (size=12001) 2024-11-20T09:01:05,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-20T09:01:06,075 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:06,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54984 deadline: 1732093326067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:06,075 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:06,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54972 deadline: 1732093326067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:06,075 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:06,076 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:06,076 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:06,076 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:06,076 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:06,076 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:06,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54954 deadline: 1732093326067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:06,076 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:06,076 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:06,076 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:06,076 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:06,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54996 deadline: 1732093326067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:06,076 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:06,076 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:06,076 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:06,076 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:06,076 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:06,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54922 deadline: 1732093326067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:06,077 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:06,077 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:06,077 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:06,159 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/.tmp/C/6bb91811e3a84dad8ac240d26209cec0 2024-11-20T09:01:06,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/.tmp/A/3ec6a601a47844448516b3baef042ca9 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/A/3ec6a601a47844448516b3baef042ca9 2024-11-20T09:01:06,167 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/A/3ec6a601a47844448516b3baef042ca9, entries=150, sequenceid=116, filesize=30.2 K 2024-11-20T09:01:06,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/.tmp/B/9e950dc9d2e4492db1716b54755bdced as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/B/9e950dc9d2e4492db1716b54755bdced 2024-11-20T09:01:06,171 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/B/9e950dc9d2e4492db1716b54755bdced, entries=150, sequenceid=116, filesize=11.7 K 2024-11-20T09:01:06,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/.tmp/C/6bb91811e3a84dad8ac240d26209cec0 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/C/6bb91811e3a84dad8ac240d26209cec0 2024-11-20T09:01:06,176 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/C/6bb91811e3a84dad8ac240d26209cec0, entries=150, sequenceid=116, filesize=11.7 K 2024-11-20T09:01:06,177 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for b3cbab3048a5449de36008b3c0424a49 in 1672ms, sequenceid=116, compaction requested=false 2024-11-20T09:01:06,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2603): Flush status journal for b3cbab3048a5449de36008b3c0424a49: 2024-11-20T09:01:06,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49. 2024-11-20T09:01:06,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=98 2024-11-20T09:01:06,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.HMaster(4169): Remote procedure done, pid=98 2024-11-20T09:01:06,182 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=98, resume processing ppid=97 2024-11-20T09:01:06,182 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=98, ppid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4390 sec 2024-11-20T09:01:06,183 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees in 2.4440 sec 2024-11-20T09:01:07,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(8855): Flush requested on b3cbab3048a5449de36008b3c0424a49 2024-11-20T09:01:07,090 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T09:01:07,090 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b3cbab3048a5449de36008b3c0424a49 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-20T09:01:07,098 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112051f8b0660fee4bb6bc32b3ddb7e73d14_b3cbab3048a5449de36008b3c0424a49 is 50, key is test_row_0/A:col10/1732093264846/Put/seqid=0 2024-11-20T09:01:07,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742237_1413 (size=14694) 2024-11-20T09:01:07,132 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:07,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54984 deadline: 1732093327125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:07,133 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:07,133 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:07,133 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:07,136 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:07,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54972 deadline: 1732093327126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:07,136 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:07,136 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:07,136 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:07,136 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:07,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54996 deadline: 1732093327126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:07,136 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:07,136 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:07,137 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:07,141 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:07,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54922 deadline: 1732093327127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:07,141 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:07,141 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:07,141 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:07,146 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:07,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54954 deadline: 1732093327133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:07,147 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:07,147 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:07,147 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:07,248 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:07,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54984 deadline: 1732093327237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:07,248 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:07,248 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:07,248 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:07,251 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:07,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54972 deadline: 1732093327246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:07,252 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:07,252 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:07,252 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:07,252 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:07,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54996 deadline: 1732093327247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:07,252 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:07,252 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:07,252 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:07,252 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:07,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54922 deadline: 1732093327247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:07,252 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:07,252 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:07,252 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:07,264 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:07,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54954 deadline: 1732093327257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:07,265 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:07,265 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:07,265 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:07,460 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:07,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54984 deadline: 1732093327457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:07,460 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:07,460 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:07,460 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:07,460 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:07,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54972 deadline: 1732093327457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:07,461 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:07,461 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:07,461 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:07,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54996 deadline: 1732093327457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:07,461 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:07,461 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:07,461 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:07,461 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:07,461 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:07,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54922 deadline: 1732093327457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:07,461 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:07,461 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:07,461 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:07,473 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:07,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54954 deadline: 1732093327467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:07,473 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:07,474 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:07,474 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:07,503 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:07,506 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112051f8b0660fee4bb6bc32b3ddb7e73d14_b3cbab3048a5449de36008b3c0424a49 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112051f8b0660fee4bb6bc32b3ddb7e73d14_b3cbab3048a5449de36008b3c0424a49 2024-11-20T09:01:07,508 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/.tmp/A/2a0066c1948a4328900bf14e6e770d58, store: [table=TestAcidGuarantees family=A region=b3cbab3048a5449de36008b3c0424a49] 2024-11-20T09:01:07,509 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/.tmp/A/2a0066c1948a4328900bf14e6e770d58 is 175, key is test_row_0/A:col10/1732093264846/Put/seqid=0 2024-11-20T09:01:07,512 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742238_1414 (size=39649) 2024-11-20T09:01:07,774 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:07,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54984 deadline: 1732093327767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:07,774 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:07,774 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:07,774 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:07,774 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:07,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54972 deadline: 1732093327767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:07,775 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:07,775 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:07,775 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:07,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54996 deadline: 1732093327767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:07,775 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:07,775 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:07,775 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:07,775 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:07,775 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:07,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54922 deadline: 1732093327768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:07,775 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:07,775 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:07,775 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:07,788 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:07,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54954 deadline: 1732093327777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:07,788 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:07,788 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:07,788 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:07,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-20T09:01:07,897 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T09:01:07,897 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T09:01:07,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] procedure2.ProcedureExecutor(1139): Stored pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees 2024-11-20T09:01:07,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-20T09:01:07,899 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T09:01:07,900 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T09:01:07,900 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=100, ppid=99, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T09:01:07,913 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=133, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/.tmp/A/2a0066c1948a4328900bf14e6e770d58 2024-11-20T09:01:07,919 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/.tmp/B/cee1924ff17d49b5b5594194e145faac is 50, key is test_row_0/B:col10/1732093264846/Put/seqid=0 2024-11-20T09:01:07,922 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742239_1415 (size=12051) 2024-11-20T09:01:08,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-20T09:01:08,052 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-20T09:01:08,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49. 2024-11-20T09:01:08,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49. as already flushing 2024-11-20T09:01:08,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49. 2024-11-20T09:01:08,052 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => b3cbab3048a5449de36008b3c0424a49, NAME => 'TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:08,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => b3cbab3048a5449de36008b3c0424a49, NAME => 'TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:08,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b3cbab3048a5449de36008b3c0424a49, NAME => 'TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b3cbab3048a5449de36008b3c0424a49, NAME => 'TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:08,205 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-20T09:01:08,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49. 2024-11-20T09:01:08,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49. as already flushing 2024-11-20T09:01:08,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49. 2024-11-20T09:01:08,206 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => b3cbab3048a5449de36008b3c0424a49, NAME => 'TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:08,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => b3cbab3048a5449de36008b3c0424a49, NAME => 'TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:08,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b3cbab3048a5449de36008b3c0424a49, NAME => 'TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b3cbab3048a5449de36008b3c0424a49, NAME => 'TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:08,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-20T09:01:08,283 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:08,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54972 deadline: 1732093328277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:08,284 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:08,284 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:08,284 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:08,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54922 deadline: 1732093328277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:08,284 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:08,284 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:08,284 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:08,284 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:08,297 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:08,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54984 deadline: 1732093328287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:08,298 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:08,298 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:08,298 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:08,298 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:08,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54996 deadline: 1732093328287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:08,298 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:08,298 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:08,298 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:08,298 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:08,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54954 deadline: 1732093328297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:08,298 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:08,298 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:08,298 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:08,322 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/.tmp/B/cee1924ff17d49b5b5594194e145faac 2024-11-20T09:01:08,329 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/.tmp/C/d8656793908c48eb89b54dc09d31f330 is 50, key is test_row_0/C:col10/1732093264846/Put/seqid=0 2024-11-20T09:01:08,332 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742240_1416 (size=12051) 2024-11-20T09:01:08,360 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-20T09:01:08,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49. 2024-11-20T09:01:08,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49. as already flushing 2024-11-20T09:01:08,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49. 2024-11-20T09:01:08,360 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => b3cbab3048a5449de36008b3c0424a49, NAME => 'TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:08,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => b3cbab3048a5449de36008b3c0424a49, NAME => 'TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:08,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b3cbab3048a5449de36008b3c0424a49, NAME => 'TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b3cbab3048a5449de36008b3c0424a49, NAME => 'TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:08,513 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-20T09:01:08,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49. 2024-11-20T09:01:08,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49. as already flushing 2024-11-20T09:01:08,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49. 2024-11-20T09:01:08,514 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => b3cbab3048a5449de36008b3c0424a49, NAME => 'TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:08,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => b3cbab3048a5449de36008b3c0424a49, NAME => 'TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:08,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b3cbab3048a5449de36008b3c0424a49, NAME => 'TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b3cbab3048a5449de36008b3c0424a49, NAME => 'TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:08,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-20T09:01:08,666 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-20T09:01:08,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49. 2024-11-20T09:01:08,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49. as already flushing 2024-11-20T09:01:08,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49. 2024-11-20T09:01:08,667 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => b3cbab3048a5449de36008b3c0424a49, NAME => 'TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:08,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => b3cbab3048a5449de36008b3c0424a49, NAME => 'TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:08,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b3cbab3048a5449de36008b3c0424a49, NAME => 'TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b3cbab3048a5449de36008b3c0424a49, NAME => 'TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:08,733 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/.tmp/C/d8656793908c48eb89b54dc09d31f330 2024-11-20T09:01:08,737 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/.tmp/A/2a0066c1948a4328900bf14e6e770d58 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/A/2a0066c1948a4328900bf14e6e770d58 2024-11-20T09:01:08,740 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/A/2a0066c1948a4328900bf14e6e770d58, entries=200, sequenceid=133, filesize=38.7 K 2024-11-20T09:01:08,741 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/.tmp/B/cee1924ff17d49b5b5594194e145faac as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/B/cee1924ff17d49b5b5594194e145faac 2024-11-20T09:01:08,745 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/B/cee1924ff17d49b5b5594194e145faac, entries=150, sequenceid=133, filesize=11.8 K 2024-11-20T09:01:08,746 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/.tmp/C/d8656793908c48eb89b54dc09d31f330 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/C/d8656793908c48eb89b54dc09d31f330 2024-11-20T09:01:08,749 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/C/d8656793908c48eb89b54dc09d31f330, entries=150, sequenceid=133, filesize=11.8 K 2024-11-20T09:01:08,750 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for b3cbab3048a5449de36008b3c0424a49 in 1660ms, sequenceid=133, compaction requested=true 2024-11-20T09:01:08,750 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b3cbab3048a5449de36008b3c0424a49: 2024-11-20T09:01:08,750 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b3cbab3048a5449de36008b3c0424a49:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T09:01:08,750 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T09:01:08,750 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b3cbab3048a5449de36008b3c0424a49:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T09:01:08,750 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T09:01:08,750 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T09:01:08,750 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b3cbab3048a5449de36008b3c0424a49:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T09:01:08,750 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T09:01:08,750 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T09:01:08,751 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101765 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T09:01:08,751 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36259 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T09:01:08,751 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1541): b3cbab3048a5449de36008b3c0424a49/B is initiating minor compaction (all files) 2024-11-20T09:01:08,751 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HStore(1541): b3cbab3048a5449de36008b3c0424a49/A is initiating minor compaction (all files) 2024-11-20T09:01:08,751 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b3cbab3048a5449de36008b3c0424a49/B in TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49. 2024-11-20T09:01:08,751 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b3cbab3048a5449de36008b3c0424a49/A in TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49. 2024-11-20T09:01:08,751 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/B/c1b661f8f45c4954ba501cb303767e92, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/B/9e950dc9d2e4492db1716b54755bdced, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/B/cee1924ff17d49b5b5594194e145faac] into tmpdir=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/.tmp, totalSize=35.4 K 2024-11-20T09:01:08,751 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/A/750522ba63754a5c80d7ce9ed131873f, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/A/3ec6a601a47844448516b3baef042ca9, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/A/2a0066c1948a4328900bf14e6e770d58] into tmpdir=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/.tmp, totalSize=99.4 K 2024-11-20T09:01:08,751 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49. 2024-11-20T09:01:08,751 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49. files: [hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/A/750522ba63754a5c80d7ce9ed131873f, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/A/3ec6a601a47844448516b3baef042ca9, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/A/2a0066c1948a4328900bf14e6e770d58] 2024-11-20T09:01:08,751 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting c1b661f8f45c4954ba501cb303767e92, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1732093262982 2024-11-20T09:01:08,751 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.Compactor(225): Compacting 750522ba63754a5c80d7ce9ed131873f, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1732093262982 2024-11-20T09:01:08,751 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting 9e950dc9d2e4492db1716b54755bdced, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1732093263666 2024-11-20T09:01:08,752 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3ec6a601a47844448516b3baef042ca9, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1732093263666 2024-11-20T09:01:08,752 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting cee1924ff17d49b5b5594194e145faac, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1732093264846 2024-11-20T09:01:08,752 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2a0066c1948a4328900bf14e6e770d58, keycount=200, bloomtype=ROW, size=38.7 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1732093264846 2024-11-20T09:01:08,757 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=b3cbab3048a5449de36008b3c0424a49] 2024-11-20T09:01:08,758 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b3cbab3048a5449de36008b3c0424a49#B#compaction#353 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-20T09:01:08,759 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/.tmp/B/7a19adabdc9c4a0690610744c6f12f8a is 50, key is test_row_0/B:col10/1732093264846/Put/seqid=0 2024-11-20T09:01:08,769 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024112020ba0f83ea4f47be8744fd93a39341e2_b3cbab3048a5449de36008b3c0424a49 store=[table=TestAcidGuarantees family=A region=b3cbab3048a5449de36008b3c0424a49] 2024-11-20T09:01:08,771 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024112020ba0f83ea4f47be8744fd93a39341e2_b3cbab3048a5449de36008b3c0424a49, store=[table=TestAcidGuarantees family=A region=b3cbab3048a5449de36008b3c0424a49] 2024-11-20T09:01:08,771 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112020ba0f83ea4f47be8744fd93a39341e2_b3cbab3048a5449de36008b3c0424a49 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=b3cbab3048a5449de36008b3c0424a49] 2024-11-20T09:01:08,773 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742241_1417 (size=12359) 2024-11-20T09:01:08,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742242_1418 (size=4469) 2024-11-20T09:01:08,778 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b3cbab3048a5449de36008b3c0424a49#A#compaction#354 average throughput is 1.16 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T09:01:08,779 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/.tmp/A/d44a8f1fb31646d28bdb7db14ccc9984 is 175, key is test_row_0/A:col10/1732093264846/Put/seqid=0 2024-11-20T09:01:08,782 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/.tmp/B/7a19adabdc9c4a0690610744c6f12f8a as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/B/7a19adabdc9c4a0690610744c6f12f8a 2024-11-20T09:01:08,787 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b3cbab3048a5449de36008b3c0424a49/B of b3cbab3048a5449de36008b3c0424a49 into 7a19adabdc9c4a0690610744c6f12f8a(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T09:01:08,787 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b3cbab3048a5449de36008b3c0424a49: 2024-11-20T09:01:08,787 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., storeName=b3cbab3048a5449de36008b3c0424a49/B, priority=13, startTime=1732093268750; duration=0sec 2024-11-20T09:01:08,787 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T09:01:08,787 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b3cbab3048a5449de36008b3c0424a49:B 2024-11-20T09:01:08,787 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T09:01:08,790 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36259 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T09:01:08,790 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1541): b3cbab3048a5449de36008b3c0424a49/C is initiating minor compaction (all files) 2024-11-20T09:01:08,790 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b3cbab3048a5449de36008b3c0424a49/C in TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49. 2024-11-20T09:01:08,791 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/C/65f9289a98334644ae0ca1227626edc6, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/C/6bb91811e3a84dad8ac240d26209cec0, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/C/d8656793908c48eb89b54dc09d31f330] into tmpdir=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/.tmp, totalSize=35.4 K 2024-11-20T09:01:08,791 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting 65f9289a98334644ae0ca1227626edc6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1732093262982 2024-11-20T09:01:08,791 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting 6bb91811e3a84dad8ac240d26209cec0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1732093263666 2024-11-20T09:01:08,791 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting d8656793908c48eb89b54dc09d31f330, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1732093264846 2024-11-20T09:01:08,795 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742243_1419 (size=31313) 2024-11-20T09:01:08,798 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b3cbab3048a5449de36008b3c0424a49#C#compaction#355 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T09:01:08,799 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/.tmp/A/d44a8f1fb31646d28bdb7db14ccc9984 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/A/d44a8f1fb31646d28bdb7db14ccc9984 2024-11-20T09:01:08,799 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/.tmp/C/1c2a5a90531a4021bfea8a1af76e90dd is 50, key is test_row_0/C:col10/1732093264846/Put/seqid=0 2024-11-20T09:01:08,804 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742244_1420 (size=12359) 2024-11-20T09:01:08,806 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b3cbab3048a5449de36008b3c0424a49/A of b3cbab3048a5449de36008b3c0424a49 into d44a8f1fb31646d28bdb7db14ccc9984(size=30.6 K), total size for store is 30.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T09:01:08,806 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b3cbab3048a5449de36008b3c0424a49: 2024-11-20T09:01:08,807 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., storeName=b3cbab3048a5449de36008b3c0424a49/A, priority=13, startTime=1732093268750; duration=0sec 2024-11-20T09:01:08,807 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T09:01:08,807 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b3cbab3048a5449de36008b3c0424a49:A 2024-11-20T09:01:08,808 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/.tmp/C/1c2a5a90531a4021bfea8a1af76e90dd as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/C/1c2a5a90531a4021bfea8a1af76e90dd 2024-11-20T09:01:08,812 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b3cbab3048a5449de36008b3c0424a49/C of b3cbab3048a5449de36008b3c0424a49 into 1c2a5a90531a4021bfea8a1af76e90dd(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T09:01:08,812 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b3cbab3048a5449de36008b3c0424a49: 2024-11-20T09:01:08,812 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., storeName=b3cbab3048a5449de36008b3c0424a49/C, priority=13, startTime=1732093268750; duration=0sec 2024-11-20T09:01:08,812 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T09:01:08,812 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b3cbab3048a5449de36008b3c0424a49:C 2024-11-20T09:01:08,819 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-20T09:01:08,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49. 2024-11-20T09:01:08,819 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2902): Flushing b3cbab3048a5449de36008b3c0424a49 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-20T09:01:08,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120ee361222653c4e4580b7f98dc86e51ed_b3cbab3048a5449de36008b3c0424a49 is 50, key is test_row_0/A:col10/1732093267131/Put/seqid=0 2024-11-20T09:01:08,835 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742245_1421 (size=12304) 2024-11-20T09:01:08,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:08,838 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120ee361222653c4e4580b7f98dc86e51ed_b3cbab3048a5449de36008b3c0424a49 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120ee361222653c4e4580b7f98dc86e51ed_b3cbab3048a5449de36008b3c0424a49 2024-11-20T09:01:08,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/.tmp/A/73de846622ff4784aa47849a6da92ccb, store: [table=TestAcidGuarantees family=A region=b3cbab3048a5449de36008b3c0424a49] 2024-11-20T09:01:08,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/.tmp/A/73de846622ff4784aa47849a6da92ccb is 175, key is test_row_0/A:col10/1732093267131/Put/seqid=0 2024-11-20T09:01:08,844 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742246_1422 (size=31105) 2024-11-20T09:01:09,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-20T09:01:09,244 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=158, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/.tmp/A/73de846622ff4784aa47849a6da92ccb 2024-11-20T09:01:09,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/.tmp/B/10fbd8399a2342c1999c414ea0e046f3 is 50, key is test_row_0/B:col10/1732093267131/Put/seqid=0 2024-11-20T09:01:09,256 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742247_1423 (size=12151) 2024-11-20T09:01:09,302 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49. as already flushing 2024-11-20T09:01:09,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(8855): Flush requested on b3cbab3048a5449de36008b3c0424a49 2024-11-20T09:01:09,338 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:09,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54984 deadline: 1732093329327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:09,339 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:09,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54972 deadline: 1732093329328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:09,339 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:09,339 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:09,339 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:09,339 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:09,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54996 deadline: 1732093329328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:09,339 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:09,339 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:09,339 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:09,339 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:09,339 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:09,339 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:09,341 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:09,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54954 deadline: 1732093329329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:09,342 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:09,342 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:09,342 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:09,342 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:09,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54922 deadline: 1732093329329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:09,342 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:09,342 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:09,342 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:09,456 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:09,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54984 deadline: 1732093329447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:09,456 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:09,456 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:09,456 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:09,456 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:09,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54996 deadline: 1732093329447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:09,457 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:09,457 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:09,457 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:09,457 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:09,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54972 deadline: 1732093329447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:09,457 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:09,457 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:09,457 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:09,457 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:09,457 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:09,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54954 deadline: 1732093329447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:09,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54922 deadline: 1732093329447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:09,457 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:09,457 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:09,457 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:09,457 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:09,457 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:09,457 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:09,657 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=158 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/.tmp/B/10fbd8399a2342c1999c414ea0e046f3 2024-11-20T09:01:09,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/.tmp/C/a6c191405a614b5e83ad8044dd99f075 is 50, key is test_row_0/C:col10/1732093267131/Put/seqid=0 2024-11-20T09:01:09,669 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742248_1424 (size=12151) 2024-11-20T09:01:09,669 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=158 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/.tmp/C/a6c191405a614b5e83ad8044dd99f075 2024-11-20T09:01:09,669 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:09,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54984 deadline: 1732093329667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:09,670 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:09,670 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:09,670 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:09,670 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:09,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54954 deadline: 1732093329668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:09,670 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:09,670 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:09,670 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:09,670 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:09,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54972 deadline: 1732093329668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:09,671 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:09,671 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:09,671 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:09,671 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:09,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54922 deadline: 1732093329668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:09,671 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:09,671 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:09,671 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:09,671 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:09,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54996 deadline: 1732093329668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:09,672 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:09,672 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:09,672 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:09,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/.tmp/A/73de846622ff4784aa47849a6da92ccb as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/A/73de846622ff4784aa47849a6da92ccb 2024-11-20T09:01:09,678 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/A/73de846622ff4784aa47849a6da92ccb, entries=150, sequenceid=158, filesize=30.4 K 2024-11-20T09:01:09,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/.tmp/B/10fbd8399a2342c1999c414ea0e046f3 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/B/10fbd8399a2342c1999c414ea0e046f3 2024-11-20T09:01:09,682 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/B/10fbd8399a2342c1999c414ea0e046f3, entries=150, sequenceid=158, filesize=11.9 K 2024-11-20T09:01:09,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/.tmp/C/a6c191405a614b5e83ad8044dd99f075 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/C/a6c191405a614b5e83ad8044dd99f075 2024-11-20T09:01:09,686 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/C/a6c191405a614b5e83ad8044dd99f075, entries=150, sequenceid=158, filesize=11.9 K 2024-11-20T09:01:09,687 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=80.51 KB/82440 for b3cbab3048a5449de36008b3c0424a49 in 868ms, sequenceid=158, compaction requested=false 2024-11-20T09:01:09,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2603): Flush status journal for b3cbab3048a5449de36008b3c0424a49: 2024-11-20T09:01:09,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49. 2024-11-20T09:01:09,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=100 2024-11-20T09:01:09,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.HMaster(4169): Remote procedure done, pid=100 2024-11-20T09:01:09,690 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=100, resume processing ppid=99 2024-11-20T09:01:09,690 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=100, ppid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7880 sec 2024-11-20T09:01:09,692 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees in 1.7920 sec 2024-11-20T09:01:09,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(8855): Flush requested on b3cbab3048a5449de36008b3c0424a49 2024-11-20T09:01:09,982 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T09:01:09,982 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b3cbab3048a5449de36008b3c0424a49 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-20T09:01:09,990 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411209b3801e284fd43b29b2d1bf22eca4837_b3cbab3048a5449de36008b3c0424a49 is 50, key is test_row_0/A:col10/1732093269325/Put/seqid=0 2024-11-20T09:01:09,994 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742249_1425 (size=14794) 2024-11-20T09:01:10,022 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:10,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54922 deadline: 1732093330013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:10,023 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:10,023 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:10,023 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:10,027 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:10,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54996 deadline: 1732093330013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:10,027 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:10,027 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:10,027 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:10,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54954 deadline: 1732093330015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:10,027 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:10,028 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:10,028 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:10,028 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:10,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54984 deadline: 1732093330015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:10,028 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:10,028 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:10,028 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:10,028 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:10,035 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:10,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54972 deadline: 1732093330022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:10,035 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:10,035 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:10,035 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:10,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-20T09:01:10,047 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T09:01:10,048 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T09:01:10,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] procedure2.ProcedureExecutor(1139): Stored pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees 2024-11-20T09:01:10,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-20T09:01:10,050 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T09:01:10,050 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T09:01:10,050 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=102, ppid=101, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T09:01:10,134 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:10,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54922 deadline: 1732093330126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:10,135 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:10,135 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:10,135 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:10,147 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:10,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54996 deadline: 1732093330137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:10,147 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:10,147 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:10,148 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:10,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54954 deadline: 1732093330137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:10,148 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:10,148 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:10,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54984 deadline: 1732093330137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:10,148 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:10,148 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:10,148 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:10,148 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:10,148 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:10,148 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:10,150 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:10,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54972 deadline: 1732093330147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:10,151 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:10,151 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:10,151 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:10,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-20T09:01:10,203 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-20T09:01:10,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49. 2024-11-20T09:01:10,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49. as already flushing 2024-11-20T09:01:10,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49. 2024-11-20T09:01:10,203 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => b3cbab3048a5449de36008b3c0424a49, NAME => 'TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:10,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => b3cbab3048a5449de36008b3c0424a49, NAME => 'TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:10,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b3cbab3048a5449de36008b3c0424a49, NAME => 'TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b3cbab3048a5449de36008b3c0424a49, NAME => 'TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:10,355 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-20T09:01:10,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49. 2024-11-20T09:01:10,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49. as already flushing 2024-11-20T09:01:10,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49. 2024-11-20T09:01:10,356 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => b3cbab3048a5449de36008b3c0424a49, NAME => 'TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:10,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => b3cbab3048a5449de36008b3c0424a49, NAME => 'TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:10,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b3cbab3048a5449de36008b3c0424a49, NAME => 'TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b3cbab3048a5449de36008b3c0424a49, NAME => 'TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:10,357 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:10,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54922 deadline: 1732093330347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:10,358 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:10,358 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:10,358 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:10,363 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:10,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54996 deadline: 1732093330357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:10,363 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:10,363 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:10,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54954 deadline: 1732093330357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:10,363 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:10,363 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:10,363 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:10,363 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:10,364 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:10,364 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:10,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54984 deadline: 1732093330357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:10,364 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:10,364 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:10,364 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:10,364 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:10,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54972 deadline: 1732093330357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:10,364 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:10,364 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:10,364 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:10,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-20T09:01:10,394 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:10,398 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411209b3801e284fd43b29b2d1bf22eca4837_b3cbab3048a5449de36008b3c0424a49 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411209b3801e284fd43b29b2d1bf22eca4837_b3cbab3048a5449de36008b3c0424a49 2024-11-20T09:01:10,399 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/.tmp/A/f967bfb0603248d194abbe372a1fcc61, store: [table=TestAcidGuarantees family=A region=b3cbab3048a5449de36008b3c0424a49] 2024-11-20T09:01:10,400 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/.tmp/A/f967bfb0603248d194abbe372a1fcc61 is 175, key is test_row_0/A:col10/1732093269325/Put/seqid=0 2024-11-20T09:01:10,403 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742250_1426 (size=39749) 2024-11-20T09:01:10,509 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-20T09:01:10,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49. 2024-11-20T09:01:10,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49. as already flushing 2024-11-20T09:01:10,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49. 2024-11-20T09:01:10,509 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => b3cbab3048a5449de36008b3c0424a49, NAME => 'TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:10,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => b3cbab3048a5449de36008b3c0424a49, NAME => 'TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:10,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b3cbab3048a5449de36008b3c0424a49, NAME => 'TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b3cbab3048a5449de36008b3c0424a49, NAME => 'TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:10,662 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-20T09:01:10,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49. 2024-11-20T09:01:10,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49. as already flushing 2024-11-20T09:01:10,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49. 2024-11-20T09:01:10,662 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => b3cbab3048a5449de36008b3c0424a49, NAME => 'TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:10,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => b3cbab3048a5449de36008b3c0424a49, NAME => 'TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:10,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b3cbab3048a5449de36008b3c0424a49, NAME => 'TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b3cbab3048a5449de36008b3c0424a49, NAME => 'TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:10,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-20T09:01:10,679 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:10,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54922 deadline: 1732093330667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:10,679 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:10,679 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:10,680 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:10,683 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:10,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54996 deadline: 1732093330677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:10,684 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:10,684 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:10,684 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:10,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54954 deadline: 1732093330677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:10,684 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:10,684 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:10,684 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:10,684 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:10,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54984 deadline: 1732093330677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:10,684 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:10,684 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:10,684 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:10,684 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:10,684 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:10,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54972 deadline: 1732093330677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:10,685 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:10,685 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:10,685 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:10,804 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=174, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/.tmp/A/f967bfb0603248d194abbe372a1fcc61 2024-11-20T09:01:10,811 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/.tmp/B/e53e8db03af14c6690a440cac19c966e is 50, key is test_row_0/B:col10/1732093269325/Put/seqid=0 2024-11-20T09:01:10,814 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742251_1427 (size=12151) 2024-11-20T09:01:10,815 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-20T09:01:10,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49. 2024-11-20T09:01:10,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49. as already flushing 2024-11-20T09:01:10,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49. 2024-11-20T09:01:10,815 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => b3cbab3048a5449de36008b3c0424a49, NAME => 'TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:10,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => b3cbab3048a5449de36008b3c0424a49, NAME => 'TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:10,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b3cbab3048a5449de36008b3c0424a49, NAME => 'TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b3cbab3048a5449de36008b3c0424a49, NAME => 'TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:10,968 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-20T09:01:10,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49. 2024-11-20T09:01:10,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49. as already flushing 2024-11-20T09:01:10,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49. 2024-11-20T09:01:10,968 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => b3cbab3048a5449de36008b3c0424a49, NAME => 'TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:10,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => b3cbab3048a5449de36008b3c0424a49, NAME => 'TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:10,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b3cbab3048a5449de36008b3c0424a49, NAME => 'TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b3cbab3048a5449de36008b3c0424a49, NAME => 'TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:11,121 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-20T09:01:11,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49. 2024-11-20T09:01:11,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49. as already flushing 2024-11-20T09:01:11,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49. 2024-11-20T09:01:11,121 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => b3cbab3048a5449de36008b3c0424a49, NAME => 'TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:11,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => b3cbab3048a5449de36008b3c0424a49, NAME => 'TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:11,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b3cbab3048a5449de36008b3c0424a49, NAME => 'TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b3cbab3048a5449de36008b3c0424a49, NAME => 'TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:11,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-20T09:01:11,194 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:11,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54922 deadline: 1732093331187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:11,195 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:11,195 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:11,195 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:11,207 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:11,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54996 deadline: 1732093331197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:11,207 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:11,208 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:11,208 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:11,208 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:11,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54954 deadline: 1732093331197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:11,208 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:11,208 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:11,208 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:11,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54972 deadline: 1732093331198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:11,208 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:11,208 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:11,208 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:11,208 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:11,208 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:11,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54984 deadline: 1732093331198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:11,209 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:11,209 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:11,209 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:11,215 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=174 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/.tmp/B/e53e8db03af14c6690a440cac19c966e 2024-11-20T09:01:11,222 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/.tmp/C/88606b297b14432a9856c1fcc106ba60 is 50, key is test_row_0/C:col10/1732093269325/Put/seqid=0 2024-11-20T09:01:11,225 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742252_1428 (size=12151) 2024-11-20T09:01:11,274 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-20T09:01:11,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49. 2024-11-20T09:01:11,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49. as already flushing 2024-11-20T09:01:11,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49. 2024-11-20T09:01:11,275 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => b3cbab3048a5449de36008b3c0424a49, NAME => 'TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:11,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => b3cbab3048a5449de36008b3c0424a49, NAME => 'TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:11,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b3cbab3048a5449de36008b3c0424a49, NAME => 'TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b3cbab3048a5449de36008b3c0424a49, NAME => 'TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:11,427 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-20T09:01:11,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49. 2024-11-20T09:01:11,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49. as already flushing 2024-11-20T09:01:11,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49. 2024-11-20T09:01:11,428 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => b3cbab3048a5449de36008b3c0424a49, NAME => 'TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:11,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => b3cbab3048a5449de36008b3c0424a49, NAME => 'TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:11,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b3cbab3048a5449de36008b3c0424a49, NAME => 'TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b3cbab3048a5449de36008b3c0424a49, NAME => 'TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:11,581 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-20T09:01:11,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49. 2024-11-20T09:01:11,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49. as already flushing 2024-11-20T09:01:11,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49. 2024-11-20T09:01:11,581 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => b3cbab3048a5449de36008b3c0424a49, NAME => 'TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:11,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => b3cbab3048a5449de36008b3c0424a49, NAME => 'TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:11,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b3cbab3048a5449de36008b3c0424a49, NAME => 'TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b3cbab3048a5449de36008b3c0424a49, NAME => 'TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:11,626 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=174 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/.tmp/C/88606b297b14432a9856c1fcc106ba60 2024-11-20T09:01:11,636 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/.tmp/A/f967bfb0603248d194abbe372a1fcc61 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/A/f967bfb0603248d194abbe372a1fcc61 2024-11-20T09:01:11,640 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/A/f967bfb0603248d194abbe372a1fcc61, entries=200, sequenceid=174, filesize=38.8 K 2024-11-20T09:01:11,642 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/.tmp/B/e53e8db03af14c6690a440cac19c966e as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/B/e53e8db03af14c6690a440cac19c966e 2024-11-20T09:01:11,648 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/B/e53e8db03af14c6690a440cac19c966e, entries=150, sequenceid=174, filesize=11.9 K 2024-11-20T09:01:11,649 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/.tmp/C/88606b297b14432a9856c1fcc106ba60 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/C/88606b297b14432a9856c1fcc106ba60 2024-11-20T09:01:11,656 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/C/88606b297b14432a9856c1fcc106ba60, entries=150, sequenceid=174, filesize=11.9 K 2024-11-20T09:01:11,657 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for b3cbab3048a5449de36008b3c0424a49 in 1675ms, sequenceid=174, compaction requested=true 2024-11-20T09:01:11,657 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b3cbab3048a5449de36008b3c0424a49: 2024-11-20T09:01:11,657 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b3cbab3048a5449de36008b3c0424a49:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T09:01:11,657 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T09:01:11,658 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b3cbab3048a5449de36008b3c0424a49:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T09:01:11,658 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-20T09:01:11,658 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b3cbab3048a5449de36008b3c0424a49:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T09:01:11,658 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-20T09:01:11,658 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T09:01:11,658 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T09:01:11,662 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102167 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T09:01:11,662 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HStore(1541): b3cbab3048a5449de36008b3c0424a49/A is initiating minor compaction (all files) 2024-11-20T09:01:11,662 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b3cbab3048a5449de36008b3c0424a49/A in TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49. 2024-11-20T09:01:11,662 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/A/d44a8f1fb31646d28bdb7db14ccc9984, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/A/73de846622ff4784aa47849a6da92ccb, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/A/f967bfb0603248d194abbe372a1fcc61] into tmpdir=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/.tmp, totalSize=99.8 K 2024-11-20T09:01:11,662 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49. 2024-11-20T09:01:11,662 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49. files: [hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/A/d44a8f1fb31646d28bdb7db14ccc9984, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/A/73de846622ff4784aa47849a6da92ccb, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/A/f967bfb0603248d194abbe372a1fcc61] 2024-11-20T09:01:11,662 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36661 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T09:01:11,663 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1541): b3cbab3048a5449de36008b3c0424a49/C is initiating minor compaction (all files) 2024-11-20T09:01:11,663 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b3cbab3048a5449de36008b3c0424a49/C in TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49. 2024-11-20T09:01:11,663 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/C/1c2a5a90531a4021bfea8a1af76e90dd, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/C/a6c191405a614b5e83ad8044dd99f075, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/C/88606b297b14432a9856c1fcc106ba60] into tmpdir=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/.tmp, totalSize=35.8 K 2024-11-20T09:01:11,663 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting 1c2a5a90531a4021bfea8a1af76e90dd, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1732093264846 2024-11-20T09:01:11,663 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.Compactor(225): Compacting d44a8f1fb31646d28bdb7db14ccc9984, keycount=150, bloomtype=ROW, size=30.6 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1732093264846 2024-11-20T09:01:11,664 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting a6c191405a614b5e83ad8044dd99f075, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1732093267115 2024-11-20T09:01:11,664 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.Compactor(225): Compacting 73de846622ff4784aa47849a6da92ccb, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1732093267115 2024-11-20T09:01:11,666 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting 88606b297b14432a9856c1fcc106ba60, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1732093269325 2024-11-20T09:01:11,666 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.Compactor(225): Compacting f967bfb0603248d194abbe372a1fcc61, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1732093269312 2024-11-20T09:01:11,672 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=b3cbab3048a5449de36008b3c0424a49] 2024-11-20T09:01:11,680 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024112039f2f35778c9437fba27f021a3539385_b3cbab3048a5449de36008b3c0424a49 store=[table=TestAcidGuarantees family=A region=b3cbab3048a5449de36008b3c0424a49] 2024-11-20T09:01:11,681 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b3cbab3048a5449de36008b3c0424a49#C#compaction#363 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-20T09:01:11,682 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/.tmp/C/e3b55ae738214860a8566482ba137fae is 50, key is test_row_0/C:col10/1732093269325/Put/seqid=0 2024-11-20T09:01:11,682 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024112039f2f35778c9437fba27f021a3539385_b3cbab3048a5449de36008b3c0424a49, store=[table=TestAcidGuarantees family=A region=b3cbab3048a5449de36008b3c0424a49] 2024-11-20T09:01:11,683 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112039f2f35778c9437fba27f021a3539385_b3cbab3048a5449de36008b3c0424a49 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=b3cbab3048a5449de36008b3c0424a49] 2024-11-20T09:01:11,713 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742254_1430 (size=4469) 2024-11-20T09:01:11,726 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742253_1429 (size=12561) 2024-11-20T09:01:11,734 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-20T09:01:11,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49. 2024-11-20T09:01:11,735 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2902): Flushing b3cbab3048a5449de36008b3c0424a49 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-20T09:01:11,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411207ceff03907ca4648a719d4cfd1943cb5_b3cbab3048a5449de36008b3c0424a49 is 50, key is test_row_0/A:col10/1732093270011/Put/seqid=0 2024-11-20T09:01:11,786 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742255_1431 (size=12304) 2024-11-20T09:01:12,114 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b3cbab3048a5449de36008b3c0424a49#A#compaction#362 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T09:01:12,115 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/.tmp/A/809b7503f75c481295d517191d30290d is 175, key is test_row_0/A:col10/1732093269325/Put/seqid=0 2024-11-20T09:01:12,118 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742256_1432 (size=31515) 2024-11-20T09:01:12,130 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/.tmp/C/e3b55ae738214860a8566482ba137fae as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/C/e3b55ae738214860a8566482ba137fae 2024-11-20T09:01:12,134 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b3cbab3048a5449de36008b3c0424a49/C of b3cbab3048a5449de36008b3c0424a49 into e3b55ae738214860a8566482ba137fae(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T09:01:12,134 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b3cbab3048a5449de36008b3c0424a49: 2024-11-20T09:01:12,134 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., storeName=b3cbab3048a5449de36008b3c0424a49/C, priority=13, startTime=1732093271658; duration=0sec 2024-11-20T09:01:12,134 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T09:01:12,134 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b3cbab3048a5449de36008b3c0424a49:C 2024-11-20T09:01:12,134 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T09:01:12,135 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36661 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T09:01:12,135 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1541): b3cbab3048a5449de36008b3c0424a49/B is initiating minor compaction (all files) 2024-11-20T09:01:12,135 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b3cbab3048a5449de36008b3c0424a49/B in TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49. 2024-11-20T09:01:12,135 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/B/7a19adabdc9c4a0690610744c6f12f8a, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/B/10fbd8399a2342c1999c414ea0e046f3, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/B/e53e8db03af14c6690a440cac19c966e] into tmpdir=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/.tmp, totalSize=35.8 K 2024-11-20T09:01:12,135 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting 7a19adabdc9c4a0690610744c6f12f8a, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1732093264846 2024-11-20T09:01:12,135 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting 10fbd8399a2342c1999c414ea0e046f3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1732093267115 2024-11-20T09:01:12,136 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting e53e8db03af14c6690a440cac19c966e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1732093269325 2024-11-20T09:01:12,146 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b3cbab3048a5449de36008b3c0424a49#B#compaction#365 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T09:01:12,146 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/.tmp/B/a4f12f212082459189a30a0d44a3a0c9 is 50, key is test_row_0/B:col10/1732093269325/Put/seqid=0 2024-11-20T09:01:12,152 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742257_1433 (size=12561) 2024-11-20T09:01:12,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:12,189 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411207ceff03907ca4648a719d4cfd1943cb5_b3cbab3048a5449de36008b3c0424a49 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411207ceff03907ca4648a719d4cfd1943cb5_b3cbab3048a5449de36008b3c0424a49 2024-11-20T09:01:12,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/.tmp/A/f6028f5b83524d2aa443ab48f1a47deb, store: [table=TestAcidGuarantees family=A region=b3cbab3048a5449de36008b3c0424a49] 2024-11-20T09:01:12,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/.tmp/A/f6028f5b83524d2aa443ab48f1a47deb is 175, key is test_row_0/A:col10/1732093270011/Put/seqid=0 2024-11-20T09:01:12,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742258_1434 (size=31105) 2024-11-20T09:01:12,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-20T09:01:12,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(8855): Flush requested on b3cbab3048a5449de36008b3c0424a49 2024-11-20T09:01:12,212 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49. as already flushing 2024-11-20T09:01:12,252 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:12,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54954 deadline: 1732093332239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:12,252 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:12,252 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:12,253 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:12,253 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:12,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54922 deadline: 1732093332240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:12,253 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:12,253 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:12,253 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:12,257 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:12,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54972 deadline: 1732093332244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:12,257 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:12,257 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:12,257 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:12,257 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:12,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54984 deadline: 1732093332251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:12,257 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:12,257 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:12,258 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:12,264 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:12,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54996 deadline: 1732093332252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:12,264 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:12,264 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:12,264 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:12,366 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:12,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54954 deadline: 1732093332357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:12,367 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:12,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54922 deadline: 1732093332357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:12,368 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:12,368 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:12,368 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:12,371 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:12,371 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:12,371 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:12,377 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:12,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54972 deadline: 1732093332367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:12,378 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:12,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54984 deadline: 1732093332367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:12,378 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:12,378 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:12,378 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:12,378 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:12,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54996 deadline: 1732093332367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:12,378 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:12,378 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:12,378 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:12,379 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:12,379 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:12,379 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:12,523 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/.tmp/A/809b7503f75c481295d517191d30290d as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/A/809b7503f75c481295d517191d30290d 2024-11-20T09:01:12,527 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b3cbab3048a5449de36008b3c0424a49/A of b3cbab3048a5449de36008b3c0424a49 into 809b7503f75c481295d517191d30290d(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T09:01:12,527 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b3cbab3048a5449de36008b3c0424a49: 2024-11-20T09:01:12,527 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., storeName=b3cbab3048a5449de36008b3c0424a49/A, priority=13, startTime=1732093271657; duration=0sec 2024-11-20T09:01:12,527 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T09:01:12,527 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b3cbab3048a5449de36008b3c0424a49:A 2024-11-20T09:01:12,556 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/.tmp/B/a4f12f212082459189a30a0d44a3a0c9 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/B/a4f12f212082459189a30a0d44a3a0c9 2024-11-20T09:01:12,560 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b3cbab3048a5449de36008b3c0424a49/B of b3cbab3048a5449de36008b3c0424a49 into a4f12f212082459189a30a0d44a3a0c9(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T09:01:12,560 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b3cbab3048a5449de36008b3c0424a49: 2024-11-20T09:01:12,560 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., storeName=b3cbab3048a5449de36008b3c0424a49/B, priority=13, startTime=1732093271657; duration=0sec 2024-11-20T09:01:12,560 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T09:01:12,560 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b3cbab3048a5449de36008b3c0424a49:B 2024-11-20T09:01:12,587 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:12,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54922 deadline: 1732093332577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:12,587 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:12,587 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:12,587 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:12,587 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:12,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54954 deadline: 1732093332577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:12,588 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:12,588 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:12,588 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:12,590 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:12,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54984 deadline: 1732093332586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:12,590 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:12,590 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:12,590 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:12,590 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:12,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54996 deadline: 1732093332586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:12,591 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:12,591 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:12,591 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:12,591 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:12,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54972 deadline: 1732093332586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:12,591 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:12,591 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:12,591 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:12,594 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=194, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/.tmp/A/f6028f5b83524d2aa443ab48f1a47deb 2024-11-20T09:01:12,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/.tmp/B/3ba57674d37a436f928e84ac1df58576 is 50, key is test_row_0/B:col10/1732093270011/Put/seqid=0 2024-11-20T09:01:12,604 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742259_1435 (size=12151) 2024-11-20T09:01:12,904 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:12,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54954 deadline: 1732093332897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:12,904 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:12,904 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:12,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54984 deadline: 1732093332897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:12,904 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:12,904 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:12,904 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:12,904 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:12,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54922 deadline: 1732093332897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:12,904 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:12,905 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:12,905 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:12,905 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:12,905 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:12,905 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:12,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54996 deadline: 1732093332897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:12,905 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:12,905 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:12,905 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:12,905 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:12,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54972 deadline: 1732093332897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:12,905 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:12,905 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:12,905 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:13,005 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/.tmp/B/3ba57674d37a436f928e84ac1df58576 2024-11-20T09:01:13,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/.tmp/C/ad4d0515cc5b42c4a18aee701251e1e3 is 50, key is test_row_0/C:col10/1732093270011/Put/seqid=0 2024-11-20T09:01:13,016 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742260_1436 (size=12151) 2024-11-20T09:01:13,017 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/.tmp/C/ad4d0515cc5b42c4a18aee701251e1e3 2024-11-20T09:01:13,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/.tmp/A/f6028f5b83524d2aa443ab48f1a47deb as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/A/f6028f5b83524d2aa443ab48f1a47deb 2024-11-20T09:01:13,024 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/A/f6028f5b83524d2aa443ab48f1a47deb, entries=150, sequenceid=194, filesize=30.4 K 2024-11-20T09:01:13,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/.tmp/B/3ba57674d37a436f928e84ac1df58576 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/B/3ba57674d37a436f928e84ac1df58576 2024-11-20T09:01:13,028 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/B/3ba57674d37a436f928e84ac1df58576, entries=150, sequenceid=194, filesize=11.9 K 2024-11-20T09:01:13,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/.tmp/C/ad4d0515cc5b42c4a18aee701251e1e3 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/C/ad4d0515cc5b42c4a18aee701251e1e3 2024-11-20T09:01:13,033 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/C/ad4d0515cc5b42c4a18aee701251e1e3, entries=150, sequenceid=194, filesize=11.9 K 2024-11-20T09:01:13,034 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for b3cbab3048a5449de36008b3c0424a49 in 1298ms, sequenceid=194, compaction requested=false 2024-11-20T09:01:13,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2603): Flush status journal for b3cbab3048a5449de36008b3c0424a49: 2024-11-20T09:01:13,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49. 2024-11-20T09:01:13,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=102 2024-11-20T09:01:13,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.HMaster(4169): Remote procedure done, pid=102 2024-11-20T09:01:13,036 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=102, resume processing ppid=101 2024-11-20T09:01:13,036 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=102, ppid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.9850 sec 2024-11-20T09:01:13,038 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees in 2.9890 sec 2024-11-20T09:01:13,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(8855): Flush requested on b3cbab3048a5449de36008b3c0424a49 2024-11-20T09:01:13,412 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T09:01:13,412 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b3cbab3048a5449de36008b3c0424a49 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-20T09:01:13,422 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120a97b4d583c7a4e3885dcc12c0599767b_b3cbab3048a5449de36008b3c0424a49 is 50, key is test_row_0/A:col10/1732093273410/Put/seqid=0 2024-11-20T09:01:13,427 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742261_1437 (size=14794) 2024-11-20T09:01:13,499 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:13,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54954 deadline: 1732093333479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:13,499 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:13,499 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:13,499 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:13,499 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:13,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54922 deadline: 1732093333480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:13,500 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:13,500 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:13,500 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:13,514 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:13,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54984 deadline: 1732093333496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:13,515 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:13,515 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:13,515 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:13,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54972 deadline: 1732093333498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:13,515 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:13,515 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:13,515 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:13,515 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:13,515 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:13,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54996 deadline: 1732093333499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:13,515 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:13,515 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:13,515 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:13,618 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:13,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54954 deadline: 1732093333607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:13,618 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:13,619 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:13,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54922 deadline: 1732093333607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:13,619 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:13,619 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:13,619 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:13,619 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:13,619 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:13,620 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:13,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54984 deadline: 1732093333617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:13,620 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:13,620 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:13,620 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:13,620 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:13,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54972 deadline: 1732093333617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:13,620 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:13,620 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:13,620 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:13,620 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:13,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54996 deadline: 1732093333617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:13,621 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:13,621 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:13,621 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:13,829 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:13,833 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120a97b4d583c7a4e3885dcc12c0599767b_b3cbab3048a5449de36008b3c0424a49 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120a97b4d583c7a4e3885dcc12c0599767b_b3cbab3048a5449de36008b3c0424a49 2024-11-20T09:01:13,834 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/.tmp/A/1628ba8ca5f14e818e65729f38e1305c, store: [table=TestAcidGuarantees family=A region=b3cbab3048a5449de36008b3c0424a49] 2024-11-20T09:01:13,834 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/.tmp/A/1628ba8ca5f14e818e65729f38e1305c is 175, key is test_row_0/A:col10/1732093273410/Put/seqid=0 2024-11-20T09:01:13,841 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:13,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54984 deadline: 1732093333827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:13,842 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:13,841 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:13,842 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:13,842 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:13,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54922 deadline: 1732093333827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:13,842 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:13,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54954 deadline: 1732093333827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:13,842 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:13,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54996 deadline: 1732093333827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:13,842 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:13,842 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:13,842 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:13,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54972 deadline: 1732093333828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:13,842 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:13,843 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:13,843 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:13,843 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:13,843 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:13,843 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:13,843 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:13,843 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:13,843 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:13,843 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:13,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742262_1438 (size=39749) 2024-11-20T09:01:13,854 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=214, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/.tmp/A/1628ba8ca5f14e818e65729f38e1305c 2024-11-20T09:01:13,874 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/.tmp/B/398c4070ad7746878345bfe96d28fa00 is 50, key is test_row_0/B:col10/1732093273410/Put/seqid=0 2024-11-20T09:01:13,889 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742263_1439 (size=12151) 2024-11-20T09:01:14,151 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:14,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54984 deadline: 1732093334147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:14,151 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:14,152 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:14,152 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:14,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54954 deadline: 1732093334147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:14,152 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:14,152 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:14,152 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:14,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54922 deadline: 1732093334147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:14,152 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:14,152 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:14,152 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:14,152 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:14,152 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:14,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54972 deadline: 1732093334147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:14,152 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:14,152 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:14,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54996 deadline: 1732093334147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:14,152 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:14,153 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:14,153 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:14,153 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:14,153 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:14,153 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:14,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-20T09:01:14,217 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T09:01:14,217 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T09:01:14,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] procedure2.ProcedureExecutor(1139): Stored pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees 2024-11-20T09:01:14,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-20T09:01:14,219 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T09:01:14,220 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T09:01:14,220 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=104, ppid=103, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T09:01:14,290 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=214 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/.tmp/B/398c4070ad7746878345bfe96d28fa00 2024-11-20T09:01:14,298 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/.tmp/C/54be9a659b6348e8b60855c638e33a2f is 50, key is test_row_0/C:col10/1732093273410/Put/seqid=0 2024-11-20T09:01:14,301 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742264_1440 (size=12151) 2024-11-20T09:01:14,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-20T09:01:14,372 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-20T09:01:14,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49. 2024-11-20T09:01:14,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49. as already flushing 2024-11-20T09:01:14,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49. 2024-11-20T09:01:14,373 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => b3cbab3048a5449de36008b3c0424a49, NAME => 'TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:14,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => b3cbab3048a5449de36008b3c0424a49, NAME => 'TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:14,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b3cbab3048a5449de36008b3c0424a49, NAME => 'TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b3cbab3048a5449de36008b3c0424a49, NAME => 'TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:14,526 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-20T09:01:14,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49. 2024-11-20T09:01:14,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49. as already flushing 2024-11-20T09:01:14,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49. 2024-11-20T09:01:14,526 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => b3cbab3048a5449de36008b3c0424a49, NAME => 'TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:14,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => b3cbab3048a5449de36008b3c0424a49, NAME => 'TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:14,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b3cbab3048a5449de36008b3c0424a49, NAME => 'TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b3cbab3048a5449de36008b3c0424a49, NAME => 'TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:14,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-20T09:01:14,668 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:14,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54984 deadline: 1732093334657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:14,668 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:14,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54922 deadline: 1732093334657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:14,668 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:14,668 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:14,668 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:14,668 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:14,668 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:14,668 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:14,668 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:14,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54972 deadline: 1732093334657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:14,669 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:14,669 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:14,669 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:14,669 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:14,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54996 deadline: 1732093334657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:14,669 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:14,669 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:14,669 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:14,669 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:14,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54954 deadline: 1732093334667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:14,670 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:14,670 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:14,670 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:14,679 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-20T09:01:14,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49. 2024-11-20T09:01:14,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49. as already flushing 2024-11-20T09:01:14,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49. 2024-11-20T09:01:14,679 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => b3cbab3048a5449de36008b3c0424a49, NAME => 'TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:14,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => b3cbab3048a5449de36008b3c0424a49, NAME => 'TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:14,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b3cbab3048a5449de36008b3c0424a49, NAME => 'TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b3cbab3048a5449de36008b3c0424a49, NAME => 'TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:14,702 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=214 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/.tmp/C/54be9a659b6348e8b60855c638e33a2f 2024-11-20T09:01:14,706 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/.tmp/A/1628ba8ca5f14e818e65729f38e1305c as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/A/1628ba8ca5f14e818e65729f38e1305c 2024-11-20T09:01:14,709 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/A/1628ba8ca5f14e818e65729f38e1305c, entries=200, sequenceid=214, filesize=38.8 K 2024-11-20T09:01:14,710 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/.tmp/B/398c4070ad7746878345bfe96d28fa00 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/B/398c4070ad7746878345bfe96d28fa00 2024-11-20T09:01:14,713 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/B/398c4070ad7746878345bfe96d28fa00, entries=150, sequenceid=214, filesize=11.9 K 2024-11-20T09:01:14,714 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/.tmp/C/54be9a659b6348e8b60855c638e33a2f as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/C/54be9a659b6348e8b60855c638e33a2f 2024-11-20T09:01:14,717 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/C/54be9a659b6348e8b60855c638e33a2f, entries=150, sequenceid=214, filesize=11.9 K 2024-11-20T09:01:14,717 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for b3cbab3048a5449de36008b3c0424a49 in 1305ms, sequenceid=214, compaction requested=true 2024-11-20T09:01:14,717 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b3cbab3048a5449de36008b3c0424a49: 2024-11-20T09:01:14,718 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b3cbab3048a5449de36008b3c0424a49:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T09:01:14,718 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T09:01:14,718 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b3cbab3048a5449de36008b3c0424a49:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T09:01:14,718 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T09:01:14,718 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T09:01:14,718 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b3cbab3048a5449de36008b3c0424a49:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T09:01:14,718 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T09:01:14,718 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T09:01:14,719 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T09:01:14,719 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102369 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T09:01:14,719 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1541): b3cbab3048a5449de36008b3c0424a49/B is initiating minor compaction (all files) 2024-11-20T09:01:14,719 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HStore(1541): b3cbab3048a5449de36008b3c0424a49/A is initiating minor compaction (all files) 2024-11-20T09:01:14,719 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b3cbab3048a5449de36008b3c0424a49/B in TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49. 2024-11-20T09:01:14,719 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b3cbab3048a5449de36008b3c0424a49/A in TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49. 2024-11-20T09:01:14,719 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/B/a4f12f212082459189a30a0d44a3a0c9, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/B/3ba57674d37a436f928e84ac1df58576, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/B/398c4070ad7746878345bfe96d28fa00] into tmpdir=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/.tmp, totalSize=36.0 K 2024-11-20T09:01:14,719 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/A/809b7503f75c481295d517191d30290d, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/A/f6028f5b83524d2aa443ab48f1a47deb, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/A/1628ba8ca5f14e818e65729f38e1305c] into tmpdir=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/.tmp, totalSize=100.0 K 2024-11-20T09:01:14,719 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49. 2024-11-20T09:01:14,719 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49. files: [hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/A/809b7503f75c481295d517191d30290d, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/A/f6028f5b83524d2aa443ab48f1a47deb, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/A/1628ba8ca5f14e818e65729f38e1305c] 2024-11-20T09:01:14,721 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.Compactor(225): Compacting 809b7503f75c481295d517191d30290d, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1732093269325 2024-11-20T09:01:14,721 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting a4f12f212082459189a30a0d44a3a0c9, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1732093269325 2024-11-20T09:01:14,722 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.Compactor(225): Compacting f6028f5b83524d2aa443ab48f1a47deb, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1732093270011 2024-11-20T09:01:14,722 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting 3ba57674d37a436f928e84ac1df58576, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1732093270011 2024-11-20T09:01:14,722 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1628ba8ca5f14e818e65729f38e1305c, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1732093272237 2024-11-20T09:01:14,722 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting 398c4070ad7746878345bfe96d28fa00, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1732093272237 2024-11-20T09:01:14,727 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=b3cbab3048a5449de36008b3c0424a49] 2024-11-20T09:01:14,728 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b3cbab3048a5449de36008b3c0424a49#B#compaction#371 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-20T09:01:14,729 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411200fad13725f314a5f8af4840993ce115e_b3cbab3048a5449de36008b3c0424a49 store=[table=TestAcidGuarantees family=A region=b3cbab3048a5449de36008b3c0424a49] 2024-11-20T09:01:14,729 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/.tmp/B/2ad1c5b47fec423faa8d8265ad376ec9 is 50, key is test_row_0/B:col10/1732093273410/Put/seqid=0 2024-11-20T09:01:14,731 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411200fad13725f314a5f8af4840993ce115e_b3cbab3048a5449de36008b3c0424a49, store=[table=TestAcidGuarantees family=A region=b3cbab3048a5449de36008b3c0424a49] 2024-11-20T09:01:14,731 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411200fad13725f314a5f8af4840993ce115e_b3cbab3048a5449de36008b3c0424a49 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=b3cbab3048a5449de36008b3c0424a49] 2024-11-20T09:01:14,735 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742265_1441 (size=12663) 2024-11-20T09:01:14,735 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742266_1442 (size=4469) 2024-11-20T09:01:14,736 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b3cbab3048a5449de36008b3c0424a49#A#compaction#372 average throughput is 2.71 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T09:01:14,736 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/.tmp/A/74bfb86569de4a8eb64b98a926c7ef0a is 175, key is test_row_0/A:col10/1732093273410/Put/seqid=0 2024-11-20T09:01:14,741 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742267_1443 (size=31617) 2024-11-20T09:01:14,832 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-20T09:01:14,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49. 2024-11-20T09:01:14,832 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2902): Flushing b3cbab3048a5449de36008b3c0424a49 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-20T09:01:14,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411207e3fe088262b4b45aa3727c76a313e57_b3cbab3048a5449de36008b3c0424a49 is 50, key is test_row_0/A:col10/1732093273497/Put/seqid=0 2024-11-20T09:01:14,846 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742268_1444 (size=12304) 2024-11-20T09:01:14,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-20T09:01:15,140 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/.tmp/B/2ad1c5b47fec423faa8d8265ad376ec9 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/B/2ad1c5b47fec423faa8d8265ad376ec9 2024-11-20T09:01:15,149 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b3cbab3048a5449de36008b3c0424a49/B of b3cbab3048a5449de36008b3c0424a49 into 2ad1c5b47fec423faa8d8265ad376ec9(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T09:01:15,149 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b3cbab3048a5449de36008b3c0424a49: 2024-11-20T09:01:15,149 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., storeName=b3cbab3048a5449de36008b3c0424a49/B, priority=13, startTime=1732093274718; duration=0sec 2024-11-20T09:01:15,150 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T09:01:15,150 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b3cbab3048a5449de36008b3c0424a49:B 2024-11-20T09:01:15,150 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T09:01:15,150 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/.tmp/A/74bfb86569de4a8eb64b98a926c7ef0a as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/A/74bfb86569de4a8eb64b98a926c7ef0a 2024-11-20T09:01:15,151 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T09:01:15,151 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1541): b3cbab3048a5449de36008b3c0424a49/C is initiating minor compaction (all files) 2024-11-20T09:01:15,151 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b3cbab3048a5449de36008b3c0424a49/C in TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49. 2024-11-20T09:01:15,151 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/C/e3b55ae738214860a8566482ba137fae, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/C/ad4d0515cc5b42c4a18aee701251e1e3, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/C/54be9a659b6348e8b60855c638e33a2f] into tmpdir=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/.tmp, totalSize=36.0 K 2024-11-20T09:01:15,151 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting e3b55ae738214860a8566482ba137fae, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1732093269325 2024-11-20T09:01:15,152 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting ad4d0515cc5b42c4a18aee701251e1e3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1732093270011 2024-11-20T09:01:15,152 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting 54be9a659b6348e8b60855c638e33a2f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1732093272237 2024-11-20T09:01:15,154 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b3cbab3048a5449de36008b3c0424a49/A of b3cbab3048a5449de36008b3c0424a49 into 74bfb86569de4a8eb64b98a926c7ef0a(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T09:01:15,154 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b3cbab3048a5449de36008b3c0424a49: 2024-11-20T09:01:15,154 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., storeName=b3cbab3048a5449de36008b3c0424a49/A, priority=13, startTime=1732093274717; duration=0sec 2024-11-20T09:01:15,155 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T09:01:15,155 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b3cbab3048a5449de36008b3c0424a49:A 2024-11-20T09:01:15,159 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b3cbab3048a5449de36008b3c0424a49#C#compaction#374 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T09:01:15,160 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/.tmp/C/32e182b30ab84754b0cdf0dced2bfb48 is 50, key is test_row_0/C:col10/1732093273410/Put/seqid=0 2024-11-20T09:01:15,163 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742269_1445 (size=12663) 2024-11-20T09:01:15,167 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/.tmp/C/32e182b30ab84754b0cdf0dced2bfb48 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/C/32e182b30ab84754b0cdf0dced2bfb48 2024-11-20T09:01:15,170 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b3cbab3048a5449de36008b3c0424a49/C of b3cbab3048a5449de36008b3c0424a49 into 32e182b30ab84754b0cdf0dced2bfb48(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T09:01:15,170 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b3cbab3048a5449de36008b3c0424a49: 2024-11-20T09:01:15,170 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., storeName=b3cbab3048a5449de36008b3c0424a49/C, priority=13, startTime=1732093274718; duration=0sec 2024-11-20T09:01:15,170 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T09:01:15,170 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b3cbab3048a5449de36008b3c0424a49:C 2024-11-20T09:01:15,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:15,250 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411207e3fe088262b4b45aa3727c76a313e57_b3cbab3048a5449de36008b3c0424a49 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411207e3fe088262b4b45aa3727c76a313e57_b3cbab3048a5449de36008b3c0424a49 2024-11-20T09:01:15,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/.tmp/A/908e456b3a0147dca47bcdbcdeb95d0a, store: [table=TestAcidGuarantees family=A region=b3cbab3048a5449de36008b3c0424a49] 2024-11-20T09:01:15,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/.tmp/A/908e456b3a0147dca47bcdbcdeb95d0a is 175, key is test_row_0/A:col10/1732093273497/Put/seqid=0 2024-11-20T09:01:15,255 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742270_1446 (size=31105) 2024-11-20T09:01:15,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-20T09:01:15,656 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=233, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/.tmp/A/908e456b3a0147dca47bcdbcdeb95d0a 2024-11-20T09:01:15,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/.tmp/B/1fe063577e3048ff83a19222e31ca9f5 is 50, key is test_row_0/B:col10/1732093273497/Put/seqid=0 2024-11-20T09:01:15,672 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742271_1447 (size=12151) 2024-11-20T09:01:15,672 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=233 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/.tmp/B/1fe063577e3048ff83a19222e31ca9f5 2024-11-20T09:01:15,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/.tmp/C/50c80609bcf543498618367f3794ba4a is 50, key is test_row_0/C:col10/1732093273497/Put/seqid=0 2024-11-20T09:01:15,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(8855): Flush requested on b3cbab3048a5449de36008b3c0424a49 2024-11-20T09:01:15,681 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49. as already flushing 2024-11-20T09:01:15,691 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742272_1448 (size=12151) 2024-11-20T09:01:15,708 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:15,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54996 deadline: 1732093335696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:15,708 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:15,708 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:15,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54972 deadline: 1732093335699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:15,708 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:15,708 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:15,709 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:15,709 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:15,709 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:15,714 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:15,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54954 deadline: 1732093335706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:15,715 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:15,715 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:15,715 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:15,717 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:15,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54984 deadline: 1732093335708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:15,718 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:15,718 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:15,718 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:15,718 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:15,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54922 deadline: 1732093335708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:15,718 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:15,718 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:15,718 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:15,826 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:15,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54996 deadline: 1732093335817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:15,826 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:15,826 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:15,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54972 deadline: 1732093335817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:15,826 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:15,826 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:15,827 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:15,827 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:15,827 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:15,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54954 deadline: 1732093335817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:15,827 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:15,827 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:15,827 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:15,827 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:15,833 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:15,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54984 deadline: 1732093335827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:15,833 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:15,833 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:15,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54922 deadline: 1732093335827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:15,833 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:15,833 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:15,834 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:15,834 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:15,834 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:16,044 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:16,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54996 deadline: 1732093336037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:16,045 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:16,045 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:16,045 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:16,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54972 deadline: 1732093336037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:16,045 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:16,045 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:16,045 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:16,045 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:16,045 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:16,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54954 deadline: 1732093336037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:16,045 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:16,045 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:16,045 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:16,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54984 deadline: 1732093336037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:16,046 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:16,046 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:16,046 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:16,046 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:16,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54922 deadline: 1732093336037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:16,046 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:16,046 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:16,046 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:16,046 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:16,092 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=233 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/.tmp/C/50c80609bcf543498618367f3794ba4a 2024-11-20T09:01:16,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/.tmp/A/908e456b3a0147dca47bcdbcdeb95d0a as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/A/908e456b3a0147dca47bcdbcdeb95d0a 2024-11-20T09:01:16,102 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/A/908e456b3a0147dca47bcdbcdeb95d0a, entries=150, sequenceid=233, filesize=30.4 K 2024-11-20T09:01:16,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/.tmp/B/1fe063577e3048ff83a19222e31ca9f5 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/B/1fe063577e3048ff83a19222e31ca9f5 2024-11-20T09:01:16,106 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/B/1fe063577e3048ff83a19222e31ca9f5, entries=150, sequenceid=233, filesize=11.9 K 2024-11-20T09:01:16,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/.tmp/C/50c80609bcf543498618367f3794ba4a as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/C/50c80609bcf543498618367f3794ba4a 2024-11-20T09:01:16,111 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/C/50c80609bcf543498618367f3794ba4a, entries=150, sequenceid=233, filesize=11.9 K 2024-11-20T09:01:16,111 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=93.93 KB/96180 for b3cbab3048a5449de36008b3c0424a49 in 1279ms, sequenceid=233, compaction requested=false 2024-11-20T09:01:16,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2603): Flush status journal for b3cbab3048a5449de36008b3c0424a49: 2024-11-20T09:01:16,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49. 2024-11-20T09:01:16,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=104 2024-11-20T09:01:16,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.HMaster(4169): Remote procedure done, pid=104 2024-11-20T09:01:16,114 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=104, resume processing ppid=103 2024-11-20T09:01:16,114 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=104, ppid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8930 sec 2024-11-20T09:01:16,116 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees in 1.8980 sec 2024-11-20T09:01:16,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(8855): Flush requested on b3cbab3048a5449de36008b3c0424a49 2024-11-20T09:01:16,363 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T09:01:16,363 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b3cbab3048a5449de36008b3c0424a49 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-20T09:01:16,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-20T09:01:16,367 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T09:01:16,368 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T09:01:16,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] procedure2.ProcedureExecutor(1139): Stored pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees 2024-11-20T09:01:16,370 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T09:01:16,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-20T09:01:16,370 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T09:01:16,370 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=106, ppid=105, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T09:01:16,371 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120638c0807582441e18db54e1f928f428c_b3cbab3048a5449de36008b3c0424a49 is 50, key is test_row_0/A:col10/1732093275706/Put/seqid=0 2024-11-20T09:01:16,375 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742273_1449 (size=14794) 2024-11-20T09:01:16,399 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:16,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54984 deadline: 1732093336384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:16,399 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:16,399 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:16,399 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:16,411 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:16,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54954 deadline: 1732093336398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:16,411 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:16,411 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:16,411 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:16,411 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:16,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54922 deadline: 1732093336399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:16,411 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:16,411 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:16,411 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:16,413 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:16,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54996 deadline: 1732093336399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:16,414 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:16,414 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:16,414 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:16,414 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:16,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54972 deadline: 1732093336400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:16,414 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:16,414 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:16,414 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:16,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-20T09:01:16,518 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:16,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54984 deadline: 1732093336507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:16,519 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:16,519 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:16,519 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:16,523 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-20T09:01:16,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49. 2024-11-20T09:01:16,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49. as already flushing 2024-11-20T09:01:16,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49. 2024-11-20T09:01:16,523 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => b3cbab3048a5449de36008b3c0424a49, NAME => 'TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:16,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => b3cbab3048a5449de36008b3c0424a49, NAME => 'TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:16,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b3cbab3048a5449de36008b3c0424a49, NAME => 'TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b3cbab3048a5449de36008b3c0424a49, NAME => 'TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:16,530 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:16,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54954 deadline: 1732093336516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:16,530 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:16,530 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:16,530 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:16,530 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:16,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54922 deadline: 1732093336517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:16,531 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:16,531 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:16,531 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:16,531 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:16,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54996 deadline: 1732093336517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:16,531 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:16,532 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:16,532 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:16,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54972 deadline: 1732093336517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:16,532 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:16,532 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:16,532 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:16,532 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:16,676 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-20T09:01:16,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49. 2024-11-20T09:01:16,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49. as already flushing 2024-11-20T09:01:16,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49. 2024-11-20T09:01:16,676 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => b3cbab3048a5449de36008b3c0424a49, NAME => 'TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:16,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => b3cbab3048a5449de36008b3c0424a49, NAME => 'TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:16,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b3cbab3048a5449de36008b3c0424a49, NAME => 'TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b3cbab3048a5449de36008b3c0424a49, NAME => 'TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:16,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-20T09:01:16,732 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:16,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54984 deadline: 1732093336727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:16,733 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:16,733 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:16,733 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:16,745 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:16,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54954 deadline: 1732093336737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:16,745 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:16,745 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:16,745 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:16,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54922 deadline: 1732093336737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:16,745 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:16,745 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:16,745 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:16,745 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:16,745 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:16,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54996 deadline: 1732093336737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:16,746 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:16,746 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:16,746 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:16,746 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:16,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54972 deadline: 1732093336737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:16,746 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:16,746 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:16,746 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:16,776 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:16,779 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120638c0807582441e18db54e1f928f428c_b3cbab3048a5449de36008b3c0424a49 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120638c0807582441e18db54e1f928f428c_b3cbab3048a5449de36008b3c0424a49 2024-11-20T09:01:16,780 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/.tmp/A/3d77942e9c94435c82b75d5bbe8f2d62, store: [table=TestAcidGuarantees family=A region=b3cbab3048a5449de36008b3c0424a49] 2024-11-20T09:01:16,781 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/.tmp/A/3d77942e9c94435c82b75d5bbe8f2d62 is 175, key is test_row_0/A:col10/1732093275706/Put/seqid=0 2024-11-20T09:01:16,784 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742274_1450 (size=39749) 2024-11-20T09:01:16,828 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-20T09:01:16,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49. 2024-11-20T09:01:16,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49. as already flushing 2024-11-20T09:01:16,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49. 2024-11-20T09:01:16,829 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => b3cbab3048a5449de36008b3c0424a49, NAME => 'TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:16,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => b3cbab3048a5449de36008b3c0424a49, NAME => 'TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:16,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b3cbab3048a5449de36008b3c0424a49, NAME => 'TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b3cbab3048a5449de36008b3c0424a49, NAME => 'TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:16,982 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-20T09:01:16,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49. 2024-11-20T09:01:16,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49. as already flushing 2024-11-20T09:01:16,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49. 2024-11-20T09:01:16,982 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => b3cbab3048a5449de36008b3c0424a49, NAME => 'TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:16,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => b3cbab3048a5449de36008b3c0424a49, NAME => 'TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:16,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b3cbab3048a5449de36008b3c0424a49, NAME => 'TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b3cbab3048a5449de36008b3c0424a49, NAME => 'TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:16,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-20T09:01:17,047 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:17,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54984 deadline: 1732093337037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:17,049 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:17,049 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:17,049 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:17,054 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:17,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54954 deadline: 1732093337047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:17,055 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:17,055 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:17,055 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:17,066 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:17,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54922 deadline: 1732093337057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:17,066 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:17,066 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:17,066 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:17,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54996 deadline: 1732093337057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:17,067 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:17,067 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:17,067 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:17,067 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:17,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54972 deadline: 1732093337057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:17,067 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:17,067 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:17,067 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:17,067 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:17,135 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-20T09:01:17,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49. 2024-11-20T09:01:17,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49. as already flushing 2024-11-20T09:01:17,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49. 2024-11-20T09:01:17,136 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => b3cbab3048a5449de36008b3c0424a49, NAME => 'TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:17,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => b3cbab3048a5449de36008b3c0424a49, NAME => 'TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:17,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b3cbab3048a5449de36008b3c0424a49, NAME => 'TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b3cbab3048a5449de36008b3c0424a49, NAME => 'TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:17,185 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=254, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/.tmp/A/3d77942e9c94435c82b75d5bbe8f2d62 2024-11-20T09:01:17,192 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/.tmp/B/0a28407d98af4b769d82e3ce12424433 is 50, key is test_row_0/B:col10/1732093275706/Put/seqid=0 2024-11-20T09:01:17,196 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742275_1451 (size=12151) 2024-11-20T09:01:17,288 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-20T09:01:17,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49. 2024-11-20T09:01:17,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49. as already flushing 2024-11-20T09:01:17,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49. 2024-11-20T09:01:17,289 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => b3cbab3048a5449de36008b3c0424a49, NAME => 'TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:17,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => b3cbab3048a5449de36008b3c0424a49, NAME => 'TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:17,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b3cbab3048a5449de36008b3c0424a49, NAME => 'TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b3cbab3048a5449de36008b3c0424a49, NAME => 'TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:17,443 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-20T09:01:17,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49. 2024-11-20T09:01:17,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49. as already flushing 2024-11-20T09:01:17,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49. 2024-11-20T09:01:17,443 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => b3cbab3048a5449de36008b3c0424a49, NAME => 'TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:17,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => b3cbab3048a5449de36008b3c0424a49, NAME => 'TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:17,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b3cbab3048a5449de36008b3c0424a49, NAME => 'TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b3cbab3048a5449de36008b3c0424a49, NAME => 'TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:17,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-20T09:01:17,565 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:17,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54984 deadline: 1732093337557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:17,565 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:17,565 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:17,565 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:17,565 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:17,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54954 deadline: 1732093337557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:17,566 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:17,566 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:17,566 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:17,583 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:17,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54922 deadline: 1732093337577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:17,583 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:17,583 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:17,584 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:17,584 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:17,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54996 deadline: 1732093337577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:17,584 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:17,584 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:17,584 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:17,584 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:17,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54972 deadline: 1732093337577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:17,584 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:17,584 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:17,584 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:17,596 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=254 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/.tmp/B/0a28407d98af4b769d82e3ce12424433 2024-11-20T09:01:17,597 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-20T09:01:17,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49. 2024-11-20T09:01:17,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49. as already flushing 2024-11-20T09:01:17,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49. 2024-11-20T09:01:17,597 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => b3cbab3048a5449de36008b3c0424a49, NAME => 'TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:17,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => b3cbab3048a5449de36008b3c0424a49, NAME => 'TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:17,604 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/.tmp/C/34e143a620204bf49da26d2c9e3fd392 is 50, key is test_row_0/C:col10/1732093275706/Put/seqid=0 2024-11-20T09:01:17,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b3cbab3048a5449de36008b3c0424a49, NAME => 'TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b3cbab3048a5449de36008b3c0424a49, NAME => 'TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:17,616 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742276_1452 (size=12151) 2024-11-20T09:01:17,620 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=254 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/.tmp/C/34e143a620204bf49da26d2c9e3fd392 2024-11-20T09:01:17,625 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/.tmp/A/3d77942e9c94435c82b75d5bbe8f2d62 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/A/3d77942e9c94435c82b75d5bbe8f2d62 2024-11-20T09:01:17,629 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/A/3d77942e9c94435c82b75d5bbe8f2d62, entries=200, sequenceid=254, filesize=38.8 K 2024-11-20T09:01:17,630 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/.tmp/B/0a28407d98af4b769d82e3ce12424433 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/B/0a28407d98af4b769d82e3ce12424433 2024-11-20T09:01:17,636 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/B/0a28407d98af4b769d82e3ce12424433, entries=150, sequenceid=254, filesize=11.9 K 2024-11-20T09:01:17,637 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/.tmp/C/34e143a620204bf49da26d2c9e3fd392 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/C/34e143a620204bf49da26d2c9e3fd392 2024-11-20T09:01:17,641 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/C/34e143a620204bf49da26d2c9e3fd392, entries=150, sequenceid=254, filesize=11.9 K 2024-11-20T09:01:17,642 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=107.34 KB/109920 for b3cbab3048a5449de36008b3c0424a49 in 1279ms, sequenceid=254, compaction requested=true 2024-11-20T09:01:17,642 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b3cbab3048a5449de36008b3c0424a49: 2024-11-20T09:01:17,643 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T09:01:17,643 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b3cbab3048a5449de36008b3c0424a49:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T09:01:17,643 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T09:01:17,643 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b3cbab3048a5449de36008b3c0424a49:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T09:01:17,643 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T09:01:17,643 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T09:01:17,643 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b3cbab3048a5449de36008b3c0424a49:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T09:01:17,643 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T09:01:17,644 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102471 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T09:01:17,644 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HStore(1541): b3cbab3048a5449de36008b3c0424a49/A is initiating minor compaction (all files) 2024-11-20T09:01:17,644 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b3cbab3048a5449de36008b3c0424a49/A in TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49. 2024-11-20T09:01:17,644 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/A/74bfb86569de4a8eb64b98a926c7ef0a, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/A/908e456b3a0147dca47bcdbcdeb95d0a, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/A/3d77942e9c94435c82b75d5bbe8f2d62] into tmpdir=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/.tmp, totalSize=100.1 K 2024-11-20T09:01:17,644 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49. 2024-11-20T09:01:17,644 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49. files: [hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/A/74bfb86569de4a8eb64b98a926c7ef0a, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/A/908e456b3a0147dca47bcdbcdeb95d0a, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/A/3d77942e9c94435c82b75d5bbe8f2d62] 2024-11-20T09:01:17,647 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T09:01:17,647 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1541): b3cbab3048a5449de36008b3c0424a49/B is initiating minor compaction (all files) 2024-11-20T09:01:17,647 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b3cbab3048a5449de36008b3c0424a49/B in TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49. 2024-11-20T09:01:17,647 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/B/2ad1c5b47fec423faa8d8265ad376ec9, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/B/1fe063577e3048ff83a19222e31ca9f5, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/B/0a28407d98af4b769d82e3ce12424433] into tmpdir=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/.tmp, totalSize=36.1 K 2024-11-20T09:01:17,647 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.Compactor(225): Compacting 74bfb86569de4a8eb64b98a926c7ef0a, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1732093272237 2024-11-20T09:01:17,648 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting 2ad1c5b47fec423faa8d8265ad376ec9, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1732093272237 2024-11-20T09:01:17,648 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.Compactor(225): Compacting 908e456b3a0147dca47bcdbcdeb95d0a, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1732093273478 2024-11-20T09:01:17,650 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting 1fe063577e3048ff83a19222e31ca9f5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1732093273478 2024-11-20T09:01:17,650 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3d77942e9c94435c82b75d5bbe8f2d62, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1732093275697 2024-11-20T09:01:17,651 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting 0a28407d98af4b769d82e3ce12424433, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1732093275706 2024-11-20T09:01:17,671 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b3cbab3048a5449de36008b3c0424a49#B#compaction#380 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T09:01:17,672 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/.tmp/B/eb8180d411144839adb2b5450c506f7a is 50, key is test_row_0/B:col10/1732093275706/Put/seqid=0 2024-11-20T09:01:17,679 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=b3cbab3048a5449de36008b3c0424a49] 2024-11-20T09:01:17,684 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411202c9c16cfab664172aba98a19e6bc0ac4_b3cbab3048a5449de36008b3c0424a49 store=[table=TestAcidGuarantees family=A region=b3cbab3048a5449de36008b3c0424a49] 2024-11-20T09:01:17,686 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411202c9c16cfab664172aba98a19e6bc0ac4_b3cbab3048a5449de36008b3c0424a49, store=[table=TestAcidGuarantees family=A region=b3cbab3048a5449de36008b3c0424a49] 2024-11-20T09:01:17,686 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411202c9c16cfab664172aba98a19e6bc0ac4_b3cbab3048a5449de36008b3c0424a49 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=b3cbab3048a5449de36008b3c0424a49] 2024-11-20T09:01:17,708 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742277_1453 (size=12765) 2024-11-20T09:01:17,714 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/.tmp/B/eb8180d411144839adb2b5450c506f7a as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/B/eb8180d411144839adb2b5450c506f7a 2024-11-20T09:01:17,721 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b3cbab3048a5449de36008b3c0424a49/B of b3cbab3048a5449de36008b3c0424a49 into eb8180d411144839adb2b5450c506f7a(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T09:01:17,721 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b3cbab3048a5449de36008b3c0424a49: 2024-11-20T09:01:17,721 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., storeName=b3cbab3048a5449de36008b3c0424a49/B, priority=13, startTime=1732093277643; duration=0sec 2024-11-20T09:01:17,721 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T09:01:17,721 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b3cbab3048a5449de36008b3c0424a49:B 2024-11-20T09:01:17,721 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T09:01:17,722 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T09:01:17,722 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1541): b3cbab3048a5449de36008b3c0424a49/C is initiating minor compaction (all files) 2024-11-20T09:01:17,722 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b3cbab3048a5449de36008b3c0424a49/C in TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49. 2024-11-20T09:01:17,722 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/C/32e182b30ab84754b0cdf0dced2bfb48, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/C/50c80609bcf543498618367f3794ba4a, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/C/34e143a620204bf49da26d2c9e3fd392] into tmpdir=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/.tmp, totalSize=36.1 K 2024-11-20T09:01:17,723 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting 32e182b30ab84754b0cdf0dced2bfb48, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1732093272237 2024-11-20T09:01:17,724 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742278_1454 (size=4469) 2024-11-20T09:01:17,725 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting 50c80609bcf543498618367f3794ba4a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1732093273478 2024-11-20T09:01:17,726 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting 34e143a620204bf49da26d2c9e3fd392, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1732093275706 2024-11-20T09:01:17,726 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b3cbab3048a5449de36008b3c0424a49#A#compaction#381 average throughput is 0.52 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T09:01:17,727 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/.tmp/A/b441ebfad30544cf84cef088e3f67d77 is 175, key is test_row_0/A:col10/1732093275706/Put/seqid=0 2024-11-20T09:01:17,749 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b3cbab3048a5449de36008b3c0424a49#C#compaction#382 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T09:01:17,750 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/.tmp/C/5d25894ce7a64bc3ae004b051699c263 is 50, key is test_row_0/C:col10/1732093275706/Put/seqid=0 2024-11-20T09:01:17,761 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-20T09:01:17,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49. 2024-11-20T09:01:17,762 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2902): Flushing b3cbab3048a5449de36008b3c0424a49 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-20T09:01:17,776 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742279_1455 (size=31719) 2024-11-20T09:01:17,781 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/.tmp/A/b441ebfad30544cf84cef088e3f67d77 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/A/b441ebfad30544cf84cef088e3f67d77 2024-11-20T09:01:17,788 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b3cbab3048a5449de36008b3c0424a49/A of b3cbab3048a5449de36008b3c0424a49 into b441ebfad30544cf84cef088e3f67d77(size=31.0 K), total size for store is 31.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T09:01:17,789 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b3cbab3048a5449de36008b3c0424a49: 2024-11-20T09:01:17,789 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., storeName=b3cbab3048a5449de36008b3c0424a49/A, priority=13, startTime=1732093277642; duration=0sec 2024-11-20T09:01:17,789 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T09:01:17,789 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b3cbab3048a5449de36008b3c0424a49:A 2024-11-20T09:01:17,806 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742280_1456 (size=12765) 2024-11-20T09:01:17,812 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/.tmp/C/5d25894ce7a64bc3ae004b051699c263 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/C/5d25894ce7a64bc3ae004b051699c263 2024-11-20T09:01:17,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120021bcf3572b84933813d272bb7b7a86c_b3cbab3048a5449de36008b3c0424a49 is 50, key is test_row_0/A:col10/1732093276398/Put/seqid=0 2024-11-20T09:01:17,819 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b3cbab3048a5449de36008b3c0424a49/C of b3cbab3048a5449de36008b3c0424a49 into 5d25894ce7a64bc3ae004b051699c263(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T09:01:17,819 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b3cbab3048a5449de36008b3c0424a49: 2024-11-20T09:01:17,819 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., storeName=b3cbab3048a5449de36008b3c0424a49/C, priority=13, startTime=1732093277643; duration=0sec 2024-11-20T09:01:17,819 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T09:01:17,819 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b3cbab3048a5449de36008b3c0424a49:C 2024-11-20T09:01:17,861 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742281_1457 (size=12454) 2024-11-20T09:01:18,193 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-20T09:01:18,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:18,269 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120021bcf3572b84933813d272bb7b7a86c_b3cbab3048a5449de36008b3c0424a49 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120021bcf3572b84933813d272bb7b7a86c_b3cbab3048a5449de36008b3c0424a49 2024-11-20T09:01:18,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/.tmp/A/651bbcaaff1a40cfb3457fae3f95e901, store: [table=TestAcidGuarantees family=A region=b3cbab3048a5449de36008b3c0424a49] 2024-11-20T09:01:18,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/.tmp/A/651bbcaaff1a40cfb3457fae3f95e901 is 175, key is test_row_0/A:col10/1732093276398/Put/seqid=0 2024-11-20T09:01:18,297 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742282_1458 (size=31255) 2024-11-20T09:01:18,298 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=274, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/.tmp/A/651bbcaaff1a40cfb3457fae3f95e901 2024-11-20T09:01:18,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/.tmp/B/da927b59a8464ebfa1ba3ded419f21ea is 50, key is test_row_0/B:col10/1732093276398/Put/seqid=0 2024-11-20T09:01:18,350 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742283_1459 (size=12301) 2024-11-20T09:01:18,443 INFO [Thread-1722 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1722. 2024-11-20T09:01:18,443 DEBUG [Thread-1722 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T09:01:18,443 DEBUG [Thread-1722 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T09:01:18,443 DEBUG [Thread-1722 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T09:01:18,443 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T09:01:18,445 INFO [Thread-1724 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1724. 2024-11-20T09:01:18,445 DEBUG [Thread-1724 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T09:01:18,445 DEBUG [Thread-1724 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T09:01:18,445 DEBUG [Thread-1724 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T09:01:18,445 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T09:01:18,447 INFO [Thread-1726 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1726. 2024-11-20T09:01:18,447 DEBUG [Thread-1726 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T09:01:18,447 DEBUG [Thread-1726 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T09:01:18,447 DEBUG [Thread-1726 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T09:01:18,447 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T09:01:18,451 INFO [Thread-1728 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1728. 2024-11-20T09:01:18,451 DEBUG [Thread-1728 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T09:01:18,451 INFO [Thread-1730 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1730. 2024-11-20T09:01:18,451 DEBUG [Thread-1728 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T09:01:18,452 DEBUG [Thread-1730 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T09:01:18,452 DEBUG [Thread-1730 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T09:01:18,452 DEBUG [Thread-1728 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T09:01:18,452 DEBUG [Thread-1730 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T09:01:18,452 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T09:01:18,452 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T09:01:18,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-20T09:01:18,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(8855): Flush requested on b3cbab3048a5449de36008b3c0424a49 2024-11-20T09:01:18,578 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49. as already flushing 2024-11-20T09:01:18,579 INFO [Thread-1719 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1719. 2024-11-20T09:01:18,579 INFO [Thread-1711 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1711. 2024-11-20T09:01:18,579 DEBUG [Thread-1719 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T09:01:18,579 DEBUG [Thread-1711 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T09:01:18,579 DEBUG [Thread-1719 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T09:01:18,579 DEBUG [Thread-1711 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T09:01:18,579 DEBUG [Thread-1719 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T09:01:18,579 DEBUG [Thread-1711 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T09:01:18,579 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T09:01:18,579 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T09:01:18,589 INFO [Thread-1713 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1713. 2024-11-20T09:01:18,589 DEBUG [Thread-1713 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T09:01:18,589 DEBUG [Thread-1713 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T09:01:18,589 DEBUG [Thread-1713 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T09:01:18,589 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T09:01:18,598 INFO [Thread-1715 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1715. 2024-11-20T09:01:18,598 DEBUG [Thread-1715 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T09:01:18,598 DEBUG [Thread-1715 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T09:01:18,598 INFO [Thread-1717 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1717. 2024-11-20T09:01:18,598 DEBUG [Thread-1717 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T09:01:18,598 DEBUG [Thread-1715 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T09:01:18,598 DEBUG [Thread-1717 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T09:01:18,598 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T09:01:18,598 DEBUG [Thread-1717 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T09:01:18,598 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T09:01:18,751 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=274 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/.tmp/B/da927b59a8464ebfa1ba3ded419f21ea 2024-11-20T09:01:18,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/.tmp/C/c1035d19afe648ab84a215fd2f5e2b98 is 50, key is test_row_0/C:col10/1732093276398/Put/seqid=0 2024-11-20T09:01:18,763 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742284_1460 (size=12301) 2024-11-20T09:01:19,164 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=274 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/.tmp/C/c1035d19afe648ab84a215fd2f5e2b98 2024-11-20T09:01:19,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/.tmp/A/651bbcaaff1a40cfb3457fae3f95e901 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/A/651bbcaaff1a40cfb3457fae3f95e901 2024-11-20T09:01:19,173 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/A/651bbcaaff1a40cfb3457fae3f95e901, entries=150, sequenceid=274, filesize=30.5 K 2024-11-20T09:01:19,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/.tmp/B/da927b59a8464ebfa1ba3ded419f21ea as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/B/da927b59a8464ebfa1ba3ded419f21ea 2024-11-20T09:01:19,178 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/B/da927b59a8464ebfa1ba3ded419f21ea, entries=150, sequenceid=274, filesize=12.0 K 2024-11-20T09:01:19,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/.tmp/C/c1035d19afe648ab84a215fd2f5e2b98 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/C/c1035d19afe648ab84a215fd2f5e2b98 2024-11-20T09:01:19,185 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/C/c1035d19afe648ab84a215fd2f5e2b98, entries=150, sequenceid=274, filesize=12.0 K 2024-11-20T09:01:19,186 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=33.54 KB/34350 for b3cbab3048a5449de36008b3c0424a49 in 1424ms, sequenceid=274, compaction requested=false 2024-11-20T09:01:19,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2603): Flush status journal for b3cbab3048a5449de36008b3c0424a49: 2024-11-20T09:01:19,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49. 2024-11-20T09:01:19,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=106 2024-11-20T09:01:19,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.HMaster(4169): Remote procedure done, pid=106 2024-11-20T09:01:19,189 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=106, resume processing ppid=105 2024-11-20T09:01:19,189 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=106, ppid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.8170 sec 2024-11-20T09:01:19,191 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees in 2.8220 sec 2024-11-20T09:01:20,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-20T09:01:20,528 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T09:01:20,528 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-20T09:01:20,528 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 42 2024-11-20T09:01:20,528 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 47 2024-11-20T09:01:20,528 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 46 2024-11-20T09:01:20,528 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 39 2024-11-20T09:01:20,528 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 43 2024-11-20T09:01:20,528 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-20T09:01:20,528 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-20T09:01:20,528 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1192 2024-11-20T09:01:20,528 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3569 rows 2024-11-20T09:01:20,528 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1195 2024-11-20T09:01:20,528 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3584 rows 2024-11-20T09:01:20,528 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1195 2024-11-20T09:01:20,528 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3581 rows 2024-11-20T09:01:20,528 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1190 2024-11-20T09:01:20,528 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3569 rows 2024-11-20T09:01:20,528 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1191 2024-11-20T09:01:20,528 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3572 rows 2024-11-20T09:01:20,528 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-20T09:01:20,528 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobScanAtomicity(AcidGuaranteesTestBase.java:127) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-20T09:01:20,528 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T09:01:20,528 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T09:01:20,528 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T09:01:20,529 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-20T09:01:20,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] procedure2.ProcedureExecutor(1139): Stored pid=107, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-20T09:01:20,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-20T09:01:20,533 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732093280532"}]},"ts":"1732093280532"} 2024-11-20T09:01:20,534 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-20T09:01:20,534 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-20T09:01:20,535 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=108, ppid=107, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-20T09:01:20,536 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=109, ppid=108, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b3cbab3048a5449de36008b3c0424a49, UNASSIGN}] 2024-11-20T09:01:20,537 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=109, ppid=108, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b3cbab3048a5449de36008b3c0424a49, UNASSIGN 2024-11-20T09:01:20,539 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=109 updating hbase:meta row=b3cbab3048a5449de36008b3c0424a49, regionState=CLOSING, regionLocation=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:20,540 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=109, ppid=108, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=b3cbab3048a5449de36008b3c0424a49, UNASSIGN because future has completed 2024-11-20T09:01:20,541 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-20T09:01:20,541 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=110, ppid=109, state=RUNNABLE, hasLock=false; CloseRegionProcedure b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960}] 2024-11-20T09:01:20,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-20T09:01:20,694 INFO [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] handler.UnassignRegionHandler(122): Close b3cbab3048a5449de36008b3c0424a49 2024-11-20T09:01:20,694 DEBUG [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-20T09:01:20,694 DEBUG [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(1722): Closing b3cbab3048a5449de36008b3c0424a49, disabling compactions & flushes 2024-11-20T09:01:20,694 INFO [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49. 2024-11-20T09:01:20,694 DEBUG [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49. 2024-11-20T09:01:20,694 DEBUG [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49. after waiting 0 ms 2024-11-20T09:01:20,694 DEBUG [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49. 2024-11-20T09:01:20,694 INFO [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(2902): Flushing b3cbab3048a5449de36008b3c0424a49 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-20T09:01:20,700 DEBUG [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411209e7b1bdb864d484f9605a385a15552f5_b3cbab3048a5449de36008b3c0424a49 is 50, key is test_row_0/A:col10/1732093278587/Put/seqid=0 2024-11-20T09:01:20,705 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742285_1461 (size=12454) 2024-11-20T09:01:20,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-20T09:01:21,106 DEBUG [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:21,111 INFO [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411209e7b1bdb864d484f9605a385a15552f5_b3cbab3048a5449de36008b3c0424a49 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411209e7b1bdb864d484f9605a385a15552f5_b3cbab3048a5449de36008b3c0424a49 2024-11-20T09:01:21,112 DEBUG [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/.tmp/A/48e9b571d5f542378b20a45cba69e511, store: [table=TestAcidGuarantees family=A region=b3cbab3048a5449de36008b3c0424a49] 2024-11-20T09:01:21,113 DEBUG [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/.tmp/A/48e9b571d5f542378b20a45cba69e511 is 175, key is test_row_0/A:col10/1732093278587/Put/seqid=0 2024-11-20T09:01:21,118 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742286_1462 (size=31255) 2024-11-20T09:01:21,118 INFO [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=284, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/.tmp/A/48e9b571d5f542378b20a45cba69e511 2024-11-20T09:01:21,125 DEBUG [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/.tmp/B/d0c4af0ee17b4109af02eb392408d25c is 50, key is test_row_0/B:col10/1732093278587/Put/seqid=0 2024-11-20T09:01:21,130 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742287_1463 (size=12301) 2024-11-20T09:01:21,131 INFO [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=284 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/.tmp/B/d0c4af0ee17b4109af02eb392408d25c 2024-11-20T09:01:21,138 DEBUG [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/.tmp/C/db97f58c605a4849ae392f8dd2e37425 is 50, key is test_row_0/C:col10/1732093278587/Put/seqid=0 2024-11-20T09:01:21,145 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742288_1464 (size=12301) 2024-11-20T09:01:21,146 INFO [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=284 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/.tmp/C/db97f58c605a4849ae392f8dd2e37425 2024-11-20T09:01:21,151 DEBUG [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/.tmp/A/48e9b571d5f542378b20a45cba69e511 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/A/48e9b571d5f542378b20a45cba69e511 2024-11-20T09:01:21,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-20T09:01:21,159 INFO [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/A/48e9b571d5f542378b20a45cba69e511, entries=150, sequenceid=284, filesize=30.5 K 2024-11-20T09:01:21,160 DEBUG [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/.tmp/B/d0c4af0ee17b4109af02eb392408d25c as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/B/d0c4af0ee17b4109af02eb392408d25c 2024-11-20T09:01:21,164 INFO [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/B/d0c4af0ee17b4109af02eb392408d25c, entries=150, sequenceid=284, filesize=12.0 K 2024-11-20T09:01:21,164 DEBUG [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/.tmp/C/db97f58c605a4849ae392f8dd2e37425 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/C/db97f58c605a4849ae392f8dd2e37425 2024-11-20T09:01:21,168 INFO [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/C/db97f58c605a4849ae392f8dd2e37425, entries=150, sequenceid=284, filesize=12.0 K 2024-11-20T09:01:21,169 INFO [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for b3cbab3048a5449de36008b3c0424a49 in 475ms, sequenceid=284, compaction requested=true 2024-11-20T09:01:21,170 DEBUG [StoreCloser-TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/A/2968f7d8b4f4474184806fdcfba08b67, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/A/c50bbdd21fc24b409352f29ab7e76e0c, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/A/7deae91156e74fdfb33d0701c934bfa8, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/A/f244cb3d6ced47fb9ce45d828f7776b1, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/A/ca4d80643bcb432c86c3386b8e625bb1, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/A/f9dda23b19b74fe6b72fe6a20f2435a6, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/A/750522ba63754a5c80d7ce9ed131873f, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/A/3ec6a601a47844448516b3baef042ca9, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/A/2a0066c1948a4328900bf14e6e770d58, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/A/d44a8f1fb31646d28bdb7db14ccc9984, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/A/73de846622ff4784aa47849a6da92ccb, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/A/f967bfb0603248d194abbe372a1fcc61, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/A/809b7503f75c481295d517191d30290d, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/A/f6028f5b83524d2aa443ab48f1a47deb, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/A/1628ba8ca5f14e818e65729f38e1305c, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/A/74bfb86569de4a8eb64b98a926c7ef0a, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/A/908e456b3a0147dca47bcdbcdeb95d0a, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/A/3d77942e9c94435c82b75d5bbe8f2d62] to archive 2024-11-20T09:01:21,171 DEBUG [StoreCloser-TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-20T09:01:21,174 DEBUG [StoreCloser-TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/A/2968f7d8b4f4474184806fdcfba08b67 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/A/2968f7d8b4f4474184806fdcfba08b67 2024-11-20T09:01:21,175 DEBUG [StoreCloser-TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/A/c50bbdd21fc24b409352f29ab7e76e0c to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/A/c50bbdd21fc24b409352f29ab7e76e0c 2024-11-20T09:01:21,176 DEBUG [StoreCloser-TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/A/7deae91156e74fdfb33d0701c934bfa8 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/A/7deae91156e74fdfb33d0701c934bfa8 2024-11-20T09:01:21,178 DEBUG [StoreCloser-TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/A/f244cb3d6ced47fb9ce45d828f7776b1 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/A/f244cb3d6ced47fb9ce45d828f7776b1 2024-11-20T09:01:21,179 DEBUG [StoreCloser-TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/A/ca4d80643bcb432c86c3386b8e625bb1 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/A/ca4d80643bcb432c86c3386b8e625bb1 2024-11-20T09:01:21,180 DEBUG [StoreCloser-TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/A/f9dda23b19b74fe6b72fe6a20f2435a6 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/A/f9dda23b19b74fe6b72fe6a20f2435a6 2024-11-20T09:01:21,182 DEBUG [StoreCloser-TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/A/750522ba63754a5c80d7ce9ed131873f to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/A/750522ba63754a5c80d7ce9ed131873f 2024-11-20T09:01:21,183 DEBUG [StoreCloser-TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/A/3ec6a601a47844448516b3baef042ca9 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/A/3ec6a601a47844448516b3baef042ca9 2024-11-20T09:01:21,185 DEBUG [StoreCloser-TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/A/2a0066c1948a4328900bf14e6e770d58 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/A/2a0066c1948a4328900bf14e6e770d58 2024-11-20T09:01:21,186 DEBUG [StoreCloser-TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/A/d44a8f1fb31646d28bdb7db14ccc9984 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/A/d44a8f1fb31646d28bdb7db14ccc9984 2024-11-20T09:01:21,187 DEBUG [StoreCloser-TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/A/73de846622ff4784aa47849a6da92ccb to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/A/73de846622ff4784aa47849a6da92ccb 2024-11-20T09:01:21,190 DEBUG [StoreCloser-TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/A/f967bfb0603248d194abbe372a1fcc61 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/A/f967bfb0603248d194abbe372a1fcc61 2024-11-20T09:01:21,191 DEBUG [StoreCloser-TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/A/809b7503f75c481295d517191d30290d to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/A/809b7503f75c481295d517191d30290d 2024-11-20T09:01:21,192 DEBUG [StoreCloser-TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/A/f6028f5b83524d2aa443ab48f1a47deb to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/A/f6028f5b83524d2aa443ab48f1a47deb 2024-11-20T09:01:21,193 DEBUG [StoreCloser-TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/A/1628ba8ca5f14e818e65729f38e1305c to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/A/1628ba8ca5f14e818e65729f38e1305c 2024-11-20T09:01:21,194 DEBUG [StoreCloser-TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/A/74bfb86569de4a8eb64b98a926c7ef0a to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/A/74bfb86569de4a8eb64b98a926c7ef0a 2024-11-20T09:01:21,195 DEBUG [StoreCloser-TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/A/908e456b3a0147dca47bcdbcdeb95d0a to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/A/908e456b3a0147dca47bcdbcdeb95d0a 2024-11-20T09:01:21,197 DEBUG [StoreCloser-TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/A/3d77942e9c94435c82b75d5bbe8f2d62 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/A/3d77942e9c94435c82b75d5bbe8f2d62 2024-11-20T09:01:21,199 DEBUG [StoreCloser-TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/B/378d6d0381224951966cef5fef728d42, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/B/3b517e40b081499c82dec600a2230b86, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/B/bf51b2dafc2949489b704d0fd194ca28, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/B/2854423aa2e04ff29efd79196b434b5d, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/B/1a4d327de879499d9c1cc0fc29d3901f, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/B/c1b661f8f45c4954ba501cb303767e92, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/B/f00794dbca9f489494ee0e3611a486c3, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/B/9e950dc9d2e4492db1716b54755bdced, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/B/7a19adabdc9c4a0690610744c6f12f8a, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/B/cee1924ff17d49b5b5594194e145faac, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/B/10fbd8399a2342c1999c414ea0e046f3, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/B/a4f12f212082459189a30a0d44a3a0c9, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/B/e53e8db03af14c6690a440cac19c966e, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/B/3ba57674d37a436f928e84ac1df58576, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/B/2ad1c5b47fec423faa8d8265ad376ec9, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/B/398c4070ad7746878345bfe96d28fa00, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/B/1fe063577e3048ff83a19222e31ca9f5, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/B/0a28407d98af4b769d82e3ce12424433] to archive 2024-11-20T09:01:21,201 DEBUG [StoreCloser-TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-20T09:01:21,202 DEBUG [StoreCloser-TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/B/378d6d0381224951966cef5fef728d42 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/B/378d6d0381224951966cef5fef728d42 2024-11-20T09:01:21,203 DEBUG [StoreCloser-TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/B/3b517e40b081499c82dec600a2230b86 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/B/3b517e40b081499c82dec600a2230b86 2024-11-20T09:01:21,205 DEBUG [StoreCloser-TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/B/bf51b2dafc2949489b704d0fd194ca28 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/B/bf51b2dafc2949489b704d0fd194ca28 2024-11-20T09:01:21,206 DEBUG [StoreCloser-TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/B/2854423aa2e04ff29efd79196b434b5d to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/B/2854423aa2e04ff29efd79196b434b5d 2024-11-20T09:01:21,207 DEBUG [StoreCloser-TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/B/1a4d327de879499d9c1cc0fc29d3901f to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/B/1a4d327de879499d9c1cc0fc29d3901f 2024-11-20T09:01:21,209 DEBUG [StoreCloser-TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/B/c1b661f8f45c4954ba501cb303767e92 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/B/c1b661f8f45c4954ba501cb303767e92 2024-11-20T09:01:21,210 DEBUG [StoreCloser-TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/B/f00794dbca9f489494ee0e3611a486c3 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/B/f00794dbca9f489494ee0e3611a486c3 2024-11-20T09:01:21,211 DEBUG [StoreCloser-TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/B/9e950dc9d2e4492db1716b54755bdced to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/B/9e950dc9d2e4492db1716b54755bdced 2024-11-20T09:01:21,212 DEBUG [StoreCloser-TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/B/7a19adabdc9c4a0690610744c6f12f8a to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/B/7a19adabdc9c4a0690610744c6f12f8a 2024-11-20T09:01:21,213 DEBUG [StoreCloser-TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/B/cee1924ff17d49b5b5594194e145faac to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/B/cee1924ff17d49b5b5594194e145faac 2024-11-20T09:01:21,214 DEBUG [StoreCloser-TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/B/10fbd8399a2342c1999c414ea0e046f3 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/B/10fbd8399a2342c1999c414ea0e046f3 2024-11-20T09:01:21,216 DEBUG [StoreCloser-TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/B/a4f12f212082459189a30a0d44a3a0c9 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/B/a4f12f212082459189a30a0d44a3a0c9 2024-11-20T09:01:21,217 DEBUG [StoreCloser-TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/B/e53e8db03af14c6690a440cac19c966e to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/B/e53e8db03af14c6690a440cac19c966e 2024-11-20T09:01:21,218 DEBUG [StoreCloser-TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/B/3ba57674d37a436f928e84ac1df58576 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/B/3ba57674d37a436f928e84ac1df58576 2024-11-20T09:01:21,220 DEBUG [StoreCloser-TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/B/2ad1c5b47fec423faa8d8265ad376ec9 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/B/2ad1c5b47fec423faa8d8265ad376ec9 2024-11-20T09:01:21,221 DEBUG [StoreCloser-TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/B/398c4070ad7746878345bfe96d28fa00 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/B/398c4070ad7746878345bfe96d28fa00 2024-11-20T09:01:21,222 DEBUG [StoreCloser-TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/B/1fe063577e3048ff83a19222e31ca9f5 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/B/1fe063577e3048ff83a19222e31ca9f5 2024-11-20T09:01:21,223 DEBUG [StoreCloser-TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/B/0a28407d98af4b769d82e3ce12424433 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/B/0a28407d98af4b769d82e3ce12424433 2024-11-20T09:01:21,225 DEBUG [StoreCloser-TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/C/2b6dae35ba0c4d1287abf6f57694e7a0, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/C/46a62284bdf545599c658ae540c642f7, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/C/87f496937dcc4a2eb1092b30b9a53181, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/C/6abfa7120f714f8cb577190349335860, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/C/b00cef00e7f14b98a2fcf415f687f0a3, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/C/65f9289a98334644ae0ca1227626edc6, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/C/c4541687e2d14c8cb6f018ffe6240ac2, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/C/6bb91811e3a84dad8ac240d26209cec0, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/C/1c2a5a90531a4021bfea8a1af76e90dd, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/C/d8656793908c48eb89b54dc09d31f330, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/C/a6c191405a614b5e83ad8044dd99f075, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/C/e3b55ae738214860a8566482ba137fae, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/C/88606b297b14432a9856c1fcc106ba60, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/C/ad4d0515cc5b42c4a18aee701251e1e3, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/C/32e182b30ab84754b0cdf0dced2bfb48, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/C/54be9a659b6348e8b60855c638e33a2f, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/C/50c80609bcf543498618367f3794ba4a, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/C/34e143a620204bf49da26d2c9e3fd392] to archive 2024-11-20T09:01:21,226 DEBUG [StoreCloser-TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-20T09:01:21,227 DEBUG [StoreCloser-TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/C/2b6dae35ba0c4d1287abf6f57694e7a0 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/C/2b6dae35ba0c4d1287abf6f57694e7a0 2024-11-20T09:01:21,228 DEBUG [StoreCloser-TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/C/46a62284bdf545599c658ae540c642f7 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/C/46a62284bdf545599c658ae540c642f7 2024-11-20T09:01:21,229 DEBUG [StoreCloser-TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/C/87f496937dcc4a2eb1092b30b9a53181 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/C/87f496937dcc4a2eb1092b30b9a53181 2024-11-20T09:01:21,230 DEBUG [StoreCloser-TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/C/6abfa7120f714f8cb577190349335860 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/C/6abfa7120f714f8cb577190349335860 2024-11-20T09:01:21,231 DEBUG [StoreCloser-TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/C/b00cef00e7f14b98a2fcf415f687f0a3 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/C/b00cef00e7f14b98a2fcf415f687f0a3 2024-11-20T09:01:21,232 DEBUG [StoreCloser-TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/C/65f9289a98334644ae0ca1227626edc6 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/C/65f9289a98334644ae0ca1227626edc6 2024-11-20T09:01:21,233 DEBUG [StoreCloser-TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/C/c4541687e2d14c8cb6f018ffe6240ac2 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/C/c4541687e2d14c8cb6f018ffe6240ac2 2024-11-20T09:01:21,234 DEBUG [StoreCloser-TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/C/6bb91811e3a84dad8ac240d26209cec0 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/C/6bb91811e3a84dad8ac240d26209cec0 2024-11-20T09:01:21,235 DEBUG [StoreCloser-TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/C/1c2a5a90531a4021bfea8a1af76e90dd to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/C/1c2a5a90531a4021bfea8a1af76e90dd 2024-11-20T09:01:21,236 DEBUG [StoreCloser-TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/C/d8656793908c48eb89b54dc09d31f330 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/C/d8656793908c48eb89b54dc09d31f330 2024-11-20T09:01:21,237 DEBUG [StoreCloser-TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/C/a6c191405a614b5e83ad8044dd99f075 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/C/a6c191405a614b5e83ad8044dd99f075 2024-11-20T09:01:21,238 DEBUG [StoreCloser-TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/C/e3b55ae738214860a8566482ba137fae to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/C/e3b55ae738214860a8566482ba137fae 2024-11-20T09:01:21,239 DEBUG [StoreCloser-TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/C/88606b297b14432a9856c1fcc106ba60 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/C/88606b297b14432a9856c1fcc106ba60 2024-11-20T09:01:21,240 DEBUG [StoreCloser-TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/C/ad4d0515cc5b42c4a18aee701251e1e3 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/C/ad4d0515cc5b42c4a18aee701251e1e3 2024-11-20T09:01:21,241 DEBUG [StoreCloser-TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/C/32e182b30ab84754b0cdf0dced2bfb48 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/C/32e182b30ab84754b0cdf0dced2bfb48 2024-11-20T09:01:21,242 DEBUG [StoreCloser-TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/C/54be9a659b6348e8b60855c638e33a2f to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/C/54be9a659b6348e8b60855c638e33a2f 2024-11-20T09:01:21,243 DEBUG [StoreCloser-TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/C/50c80609bcf543498618367f3794ba4a to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/C/50c80609bcf543498618367f3794ba4a 2024-11-20T09:01:21,245 DEBUG [StoreCloser-TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/C/34e143a620204bf49da26d2c9e3fd392 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/C/34e143a620204bf49da26d2c9e3fd392 2024-11-20T09:01:21,249 DEBUG [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/recovered.edits/287.seqid, newMaxSeqId=287, maxSeqId=4 2024-11-20T09:01:21,250 INFO [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49. 2024-11-20T09:01:21,250 DEBUG [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(1676): Region close journal for b3cbab3048a5449de36008b3c0424a49: Waiting for close lock at 1732093280694Running coprocessor pre-close hooks at 1732093280694Disabling compacts and flushes for region at 1732093280694Disabling writes for close at 1732093280694Obtaining lock to block concurrent updates at 1732093280694Preparing flush snapshotting stores in b3cbab3048a5449de36008b3c0424a49 at 1732093280694Finished memstore snapshotting TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=90720, getOffHeapSize=0, getCellsCount=750 at 1732093280694Flushing stores of TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49. at 1732093280695 (+1 ms)Flushing b3cbab3048a5449de36008b3c0424a49/A: creating writer at 1732093280695Flushing mob file b3cbab3048a5449de36008b3c0424a49/A: appending metadata at 1732093280700 (+5 ms)Flushing mob file b3cbab3048a5449de36008b3c0424a49/A: closing flushed file at 1732093280700Flushing b3cbab3048a5449de36008b3c0424a49/A: appending metadata at 1732093281112 (+412 ms)Flushing b3cbab3048a5449de36008b3c0424a49/A: closing flushed file at 1732093281112Flushing b3cbab3048a5449de36008b3c0424a49/B: creating writer at 1732093281122 (+10 ms)Flushing b3cbab3048a5449de36008b3c0424a49/B: appending metadata at 1732093281125 (+3 ms)Flushing b3cbab3048a5449de36008b3c0424a49/B: closing flushed file at 1732093281125Flushing b3cbab3048a5449de36008b3c0424a49/C: creating writer at 1732093281135 (+10 ms)Flushing b3cbab3048a5449de36008b3c0424a49/C: appending metadata at 1732093281138 (+3 ms)Flushing b3cbab3048a5449de36008b3c0424a49/C: closing flushed file at 1732093281138Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7eaf6aca: reopening flushed file at 1732093281151 (+13 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@691ac52d: reopening flushed file at 1732093281159 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@34fa9cb1: reopening flushed file at 1732093281164 (+5 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for b3cbab3048a5449de36008b3c0424a49 in 475ms, sequenceid=284, compaction requested=true at 1732093281169 (+5 ms)Writing region close event to WAL at 1732093281246 (+77 ms)Running coprocessor post-close hooks at 1732093281250 (+4 ms)Closed at 1732093281250 2024-11-20T09:01:21,252 INFO [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] handler.UnassignRegionHandler(157): Closed b3cbab3048a5449de36008b3c0424a49 2024-11-20T09:01:21,253 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=109 updating hbase:meta row=b3cbab3048a5449de36008b3c0424a49, regionState=CLOSED 2024-11-20T09:01:21,255 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=110, ppid=109, state=RUNNABLE, hasLock=false; CloseRegionProcedure b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 because future has completed 2024-11-20T09:01:21,260 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=110, resume processing ppid=109 2024-11-20T09:01:21,260 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=110, ppid=109, state=SUCCESS, hasLock=false; CloseRegionProcedure b3cbab3048a5449de36008b3c0424a49, server=be1eb2620e0e,34223,1732093159960 in 714 msec 2024-11-20T09:01:21,262 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=109, resume processing ppid=108 2024-11-20T09:01:21,262 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=109, ppid=108, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b3cbab3048a5449de36008b3c0424a49, UNASSIGN in 724 msec 2024-11-20T09:01:21,265 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=108, resume processing ppid=107 2024-11-20T09:01:21,265 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=108, ppid=107, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 728 msec 2024-11-20T09:01:21,266 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732093281266"}]},"ts":"1732093281266"} 2024-11-20T09:01:21,267 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-20T09:01:21,267 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-20T09:01:21,270 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=107, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 739 msec 2024-11-20T09:01:21,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-20T09:01:21,668 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-20T09:01:21,668 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-20T09:01:21,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] procedure2.ProcedureExecutor(1139): Stored pid=111, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T09:01:21,670 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=111, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T09:01:21,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-20T09:01:21,671 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=111, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T09:01:21,680 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49 2024-11-20T09:01:21,683 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/A, FileablePath, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/B, FileablePath, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/C, FileablePath, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/recovered.edits] 2024-11-20T09:01:21,689 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/A/48e9b571d5f542378b20a45cba69e511 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/A/48e9b571d5f542378b20a45cba69e511 2024-11-20T09:01:21,691 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/A/651bbcaaff1a40cfb3457fae3f95e901 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/A/651bbcaaff1a40cfb3457fae3f95e901 2024-11-20T09:01:21,692 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/A/b441ebfad30544cf84cef088e3f67d77 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/A/b441ebfad30544cf84cef088e3f67d77 2024-11-20T09:01:21,695 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/B/d0c4af0ee17b4109af02eb392408d25c to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/B/d0c4af0ee17b4109af02eb392408d25c 2024-11-20T09:01:21,696 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/B/da927b59a8464ebfa1ba3ded419f21ea to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/B/da927b59a8464ebfa1ba3ded419f21ea 2024-11-20T09:01:21,697 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/B/eb8180d411144839adb2b5450c506f7a to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/B/eb8180d411144839adb2b5450c506f7a 2024-11-20T09:01:21,699 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/C/5d25894ce7a64bc3ae004b051699c263 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/C/5d25894ce7a64bc3ae004b051699c263 2024-11-20T09:01:21,700 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/C/c1035d19afe648ab84a215fd2f5e2b98 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/C/c1035d19afe648ab84a215fd2f5e2b98 2024-11-20T09:01:21,701 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/C/db97f58c605a4849ae392f8dd2e37425 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/C/db97f58c605a4849ae392f8dd2e37425 2024-11-20T09:01:21,704 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/recovered.edits/287.seqid to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49/recovered.edits/287.seqid 2024-11-20T09:01:21,705 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/b3cbab3048a5449de36008b3c0424a49 2024-11-20T09:01:21,705 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-20T09:01:21,706 DEBUG [PEWorker-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-20T09:01:21,707 DEBUG [PEWorker-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-20T09:01:21,710 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120021bcf3572b84933813d272bb7b7a86c_b3cbab3048a5449de36008b3c0424a49 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120021bcf3572b84933813d272bb7b7a86c_b3cbab3048a5449de36008b3c0424a49 2024-11-20T09:01:21,711 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411202302403566f44bb89dfdf15d817e6f83_b3cbab3048a5449de36008b3c0424a49 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411202302403566f44bb89dfdf15d817e6f83_b3cbab3048a5449de36008b3c0424a49 2024-11-20T09:01:21,712 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411202561b03189d44ff997bf3c54dd0914d6_b3cbab3048a5449de36008b3c0424a49 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411202561b03189d44ff997bf3c54dd0914d6_b3cbab3048a5449de36008b3c0424a49 2024-11-20T09:01:21,713 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112051f8b0660fee4bb6bc32b3ddb7e73d14_b3cbab3048a5449de36008b3c0424a49 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112051f8b0660fee4bb6bc32b3ddb7e73d14_b3cbab3048a5449de36008b3c0424a49 2024-11-20T09:01:21,714 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120638c0807582441e18db54e1f928f428c_b3cbab3048a5449de36008b3c0424a49 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120638c0807582441e18db54e1f928f428c_b3cbab3048a5449de36008b3c0424a49 2024-11-20T09:01:21,715 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411207ceff03907ca4648a719d4cfd1943cb5_b3cbab3048a5449de36008b3c0424a49 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411207ceff03907ca4648a719d4cfd1943cb5_b3cbab3048a5449de36008b3c0424a49 2024-11-20T09:01:21,716 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411207e3fe088262b4b45aa3727c76a313e57_b3cbab3048a5449de36008b3c0424a49 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411207e3fe088262b4b45aa3727c76a313e57_b3cbab3048a5449de36008b3c0424a49 2024-11-20T09:01:21,717 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411209b3801e284fd43b29b2d1bf22eca4837_b3cbab3048a5449de36008b3c0424a49 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411209b3801e284fd43b29b2d1bf22eca4837_b3cbab3048a5449de36008b3c0424a49 2024-11-20T09:01:21,718 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411209e7b1bdb864d484f9605a385a15552f5_b3cbab3048a5449de36008b3c0424a49 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411209e7b1bdb864d484f9605a385a15552f5_b3cbab3048a5449de36008b3c0424a49 2024-11-20T09:01:21,719 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120a97b4d583c7a4e3885dcc12c0599767b_b3cbab3048a5449de36008b3c0424a49 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120a97b4d583c7a4e3885dcc12c0599767b_b3cbab3048a5449de36008b3c0424a49 2024-11-20T09:01:21,720 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120aa871f8628e54960bb8f58ded37f9579_b3cbab3048a5449de36008b3c0424a49 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120aa871f8628e54960bb8f58ded37f9579_b3cbab3048a5449de36008b3c0424a49 2024-11-20T09:01:21,721 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120bf95ce65d3ed45bab803cbffaa632f39_b3cbab3048a5449de36008b3c0424a49 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120bf95ce65d3ed45bab803cbffaa632f39_b3cbab3048a5449de36008b3c0424a49 2024-11-20T09:01:21,722 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120e528d9073a1544bfa1c0a3605a2b601f_b3cbab3048a5449de36008b3c0424a49 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120e528d9073a1544bfa1c0a3605a2b601f_b3cbab3048a5449de36008b3c0424a49 2024-11-20T09:01:21,723 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120ee361222653c4e4580b7f98dc86e51ed_b3cbab3048a5449de36008b3c0424a49 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120ee361222653c4e4580b7f98dc86e51ed_b3cbab3048a5449de36008b3c0424a49 2024-11-20T09:01:21,724 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120fcb3f423fb514bf1b1c173378997044d_b3cbab3048a5449de36008b3c0424a49 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120fcb3f423fb514bf1b1c173378997044d_b3cbab3048a5449de36008b3c0424a49 2024-11-20T09:01:21,724 DEBUG [PEWorker-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-20T09:01:21,727 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=111, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T09:01:21,729 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-20T09:01:21,731 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-20T09:01:21,731 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=111, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T09:01:21,731 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-20T09:01:21,732 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732093281731"}]},"ts":"9223372036854775807"} 2024-11-20T09:01:21,734 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-20T09:01:21,734 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => b3cbab3048a5449de36008b3c0424a49, NAME => 'TestAcidGuarantees,,1732093255336.b3cbab3048a5449de36008b3c0424a49.', STARTKEY => '', ENDKEY => ''}] 2024-11-20T09:01:21,734 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-20T09:01:21,734 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732093281734"}]},"ts":"9223372036854775807"} 2024-11-20T09:01:21,736 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-20T09:01:21,736 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=111, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T09:01:21,737 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=111, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 68 msec 2024-11-20T09:01:21,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-20T09:01:21,777 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-20T09:01:21,778 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-20T09:01:21,788 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobScanAtomicity Thread=241 (was 242), OpenFileDescriptor=461 (was 459) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=412 (was 453), ProcessCount=11 (was 11), AvailableMemoryMB=2240 (was 2041) - AvailableMemoryMB LEAK? - 2024-11-20T09:01:21,798 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testScanAtomicity Thread=241, OpenFileDescriptor=461, MaxFileDescriptor=1048576, SystemLoadAverage=412, ProcessCount=11, AvailableMemoryMB=2239 2024-11-20T09:01:21,800 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-20T09:01:21,800 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-20T09:01:21,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] procedure2.ProcedureExecutor(1139): Stored pid=112, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-20T09:01:21,802 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=112, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-20T09:01:21,802 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:21,802 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 112 2024-11-20T09:01:21,803 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=112, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-20T09:01:21,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=112 2024-11-20T09:01:21,809 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742289_1465 (size=959) 2024-11-20T09:01:21,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=112 2024-11-20T09:01:22,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=112 2024-11-20T09:01:22,211 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => e765ef86c58446bc8e368358023e83a1, NAME => 'TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a 2024-11-20T09:01:22,216 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742290_1466 (size=53) 2024-11-20T09:01:22,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=112 2024-11-20T09:01:22,616 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T09:01:22,616 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing e765ef86c58446bc8e368358023e83a1, disabling compactions & flushes 2024-11-20T09:01:22,616 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1. 2024-11-20T09:01:22,616 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1. 2024-11-20T09:01:22,616 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1. after waiting 0 ms 2024-11-20T09:01:22,616 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1. 2024-11-20T09:01:22,616 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1. 2024-11-20T09:01:22,617 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for e765ef86c58446bc8e368358023e83a1: Waiting for close lock at 1732093282616Disabling compacts and flushes for region at 1732093282616Disabling writes for close at 1732093282616Writing region close event to WAL at 1732093282616Closed at 1732093282616 2024-11-20T09:01:22,618 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=112, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-20T09:01:22,618 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732093282618"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732093282618"}]},"ts":"1732093282618"} 2024-11-20T09:01:22,619 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-20T09:01:22,620 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=112, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-20T09:01:22,620 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732093282620"}]},"ts":"1732093282620"} 2024-11-20T09:01:22,621 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-20T09:01:22,622 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=113, ppid=112, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e765ef86c58446bc8e368358023e83a1, ASSIGN}] 2024-11-20T09:01:22,623 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=113, ppid=112, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e765ef86c58446bc8e368358023e83a1, ASSIGN 2024-11-20T09:01:22,623 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=113, ppid=112, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e765ef86c58446bc8e368358023e83a1, ASSIGN; state=OFFLINE, location=be1eb2620e0e,34223,1732093159960; forceNewPlan=false, retain=false 2024-11-20T09:01:22,774 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=113 updating hbase:meta row=e765ef86c58446bc8e368358023e83a1, regionState=OPENING, regionLocation=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:22,776 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=113, ppid=112, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e765ef86c58446bc8e368358023e83a1, ASSIGN because future has completed 2024-11-20T09:01:22,776 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=114, ppid=113, state=RUNNABLE, hasLock=false; OpenRegionProcedure e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960}] 2024-11-20T09:01:22,931 INFO [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1. 2024-11-20T09:01:22,931 DEBUG [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(7752): Opening region: {ENCODED => e765ef86c58446bc8e368358023e83a1, NAME => 'TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.', STARTKEY => '', ENDKEY => ''} 2024-11-20T09:01:22,932 DEBUG [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees e765ef86c58446bc8e368358023e83a1 2024-11-20T09:01:22,932 DEBUG [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T09:01:22,932 DEBUG [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(7794): checking encryption for e765ef86c58446bc8e368358023e83a1 2024-11-20T09:01:22,933 DEBUG [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(7797): checking classloading for e765ef86c58446bc8e368358023e83a1 2024-11-20T09:01:22,934 INFO [StoreOpener-e765ef86c58446bc8e368358023e83a1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region e765ef86c58446bc8e368358023e83a1 2024-11-20T09:01:22,935 INFO [StoreOpener-e765ef86c58446bc8e368358023e83a1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e765ef86c58446bc8e368358023e83a1 columnFamilyName A 2024-11-20T09:01:22,936 DEBUG [StoreOpener-e765ef86c58446bc8e368358023e83a1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:22,936 INFO [StoreOpener-e765ef86c58446bc8e368358023e83a1-1 {}] regionserver.HStore(327): Store=e765ef86c58446bc8e368358023e83a1/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T09:01:22,936 INFO [StoreOpener-e765ef86c58446bc8e368358023e83a1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region e765ef86c58446bc8e368358023e83a1 2024-11-20T09:01:22,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=112 2024-11-20T09:01:22,937 INFO [StoreOpener-e765ef86c58446bc8e368358023e83a1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e765ef86c58446bc8e368358023e83a1 columnFamilyName B 2024-11-20T09:01:22,937 DEBUG [StoreOpener-e765ef86c58446bc8e368358023e83a1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:22,938 INFO [StoreOpener-e765ef86c58446bc8e368358023e83a1-1 {}] regionserver.HStore(327): Store=e765ef86c58446bc8e368358023e83a1/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T09:01:22,938 INFO [StoreOpener-e765ef86c58446bc8e368358023e83a1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region e765ef86c58446bc8e368358023e83a1 2024-11-20T09:01:22,939 INFO [StoreOpener-e765ef86c58446bc8e368358023e83a1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e765ef86c58446bc8e368358023e83a1 columnFamilyName C 2024-11-20T09:01:22,939 DEBUG [StoreOpener-e765ef86c58446bc8e368358023e83a1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:22,939 INFO [StoreOpener-e765ef86c58446bc8e368358023e83a1-1 {}] regionserver.HStore(327): Store=e765ef86c58446bc8e368358023e83a1/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T09:01:22,939 DEBUG [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(1038): replaying wal for e765ef86c58446bc8e368358023e83a1 2024-11-20T09:01:22,940 DEBUG [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1 2024-11-20T09:01:22,940 DEBUG [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1 2024-11-20T09:01:22,941 DEBUG [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(1048): stopping wal replay for e765ef86c58446bc8e368358023e83a1 2024-11-20T09:01:22,941 DEBUG [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(1060): Cleaning up temporary data for e765ef86c58446bc8e368358023e83a1 2024-11-20T09:01:22,942 DEBUG [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-20T09:01:22,943 DEBUG [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(1093): writing seq id for e765ef86c58446bc8e368358023e83a1 2024-11-20T09:01:22,945 DEBUG [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T09:01:22,945 INFO [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(1114): Opened e765ef86c58446bc8e368358023e83a1; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61892860, jitterRate=-0.07772451639175415}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-20T09:01:22,945 DEBUG [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e765ef86c58446bc8e368358023e83a1 2024-11-20T09:01:22,946 DEBUG [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(1006): Region open journal for e765ef86c58446bc8e368358023e83a1: Running coprocessor pre-open hook at 1732093282933Writing region info on filesystem at 1732093282933Initializing all the Stores at 1732093282933Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732093282933Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732093282934 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732093282934Cleaning up temporary data from old regions at 1732093282941 (+7 ms)Running coprocessor post-open hooks at 1732093282945 (+4 ms)Region opened successfully at 1732093282946 (+1 ms) 2024-11-20T09:01:22,947 INFO [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., pid=114, masterSystemTime=1732093282928 2024-11-20T09:01:22,948 DEBUG [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1. 2024-11-20T09:01:22,949 INFO [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1. 2024-11-20T09:01:22,949 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=113 updating hbase:meta row=e765ef86c58446bc8e368358023e83a1, regionState=OPEN, openSeqNum=2, regionLocation=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:22,950 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=114, ppid=113, state=RUNNABLE, hasLock=false; OpenRegionProcedure e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 because future has completed 2024-11-20T09:01:22,953 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=114, resume processing ppid=113 2024-11-20T09:01:22,953 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=114, ppid=113, state=SUCCESS, hasLock=false; OpenRegionProcedure e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 in 175 msec 2024-11-20T09:01:22,955 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=113, resume processing ppid=112 2024-11-20T09:01:22,955 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=113, ppid=112, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e765ef86c58446bc8e368358023e83a1, ASSIGN in 331 msec 2024-11-20T09:01:22,955 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=112, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-20T09:01:22,955 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732093282955"}]},"ts":"1732093282955"} 2024-11-20T09:01:22,957 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-20T09:01:22,957 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=112, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-20T09:01:22,959 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=112, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1570 sec 2024-11-20T09:01:23,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=112 2024-11-20T09:01:23,947 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-20T09:01:23,948 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72417d0f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T09:01:23,949 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request be1eb2620e0e,44373,-1 for getting cluster id 2024-11-20T09:01:23,949 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T09:01:23,950 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'caf50035-b19e-4cfd-90ea-dca559866882' 2024-11-20T09:01:23,950 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T09:01:23,950 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "caf50035-b19e-4cfd-90ea-dca559866882" 2024-11-20T09:01:23,950 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21ba13d3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T09:01:23,950 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [be1eb2620e0e,44373,-1] 2024-11-20T09:01:23,950 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T09:01:23,950 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T09:01:23,951 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40540, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T09:01:23,952 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71de4bd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T09:01:23,952 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T09:01:23,952 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=be1eb2620e0e,34223,1732093159960, seqNum=-1] 2024-11-20T09:01:23,953 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T09:01:23,953 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52794, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T09:01:23,954 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-20T09:01:23,954 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncConnectionImpl(321): The fetched master address is be1eb2620e0e,44373,1732093159150 2024-11-20T09:01:23,954 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@4d18d109 2024-11-20T09:01:23,955 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-20T09:01:23,955 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40542, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-20T09:01:23,956 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3692791b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T09:01:23,956 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request be1eb2620e0e,44373,-1 for getting cluster id 2024-11-20T09:01:23,957 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T09:01:23,957 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'caf50035-b19e-4cfd-90ea-dca559866882' 2024-11-20T09:01:23,957 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T09:01:23,957 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "caf50035-b19e-4cfd-90ea-dca559866882" 2024-11-20T09:01:23,957 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32476b50, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T09:01:23,957 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [be1eb2620e0e,44373,-1] 2024-11-20T09:01:23,958 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T09:01:23,958 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T09:01:23,958 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40552, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T09:01:23,959 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6581b4f8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T09:01:23,959 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b791feb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T09:01:23,960 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request be1eb2620e0e,44373,-1 for getting cluster id 2024-11-20T09:01:23,960 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T09:01:23,960 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'caf50035-b19e-4cfd-90ea-dca559866882' 2024-11-20T09:01:23,960 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T09:01:23,960 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "caf50035-b19e-4cfd-90ea-dca559866882" 2024-11-20T09:01:23,961 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@31e45096, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T09:01:23,961 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [be1eb2620e0e,44373,-1] 2024-11-20T09:01:23,961 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T09:01:23,961 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T09:01:23,961 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40576, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T09:01:23,962 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@38a41fa1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T09:01:23,963 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6da2e0be, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T09:01:23,963 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request be1eb2620e0e,44373,-1 for getting cluster id 2024-11-20T09:01:23,964 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T09:01:23,964 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'caf50035-b19e-4cfd-90ea-dca559866882' 2024-11-20T09:01:23,964 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T09:01:23,964 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "caf50035-b19e-4cfd-90ea-dca559866882" 2024-11-20T09:01:23,964 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21f11c9c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T09:01:23,964 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [be1eb2620e0e,44373,-1] 2024-11-20T09:01:23,964 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T09:01:23,965 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T09:01:23,965 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40590, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T09:01:23,966 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f779474, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T09:01:23,966 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3fc361f0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T09:01:23,966 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request be1eb2620e0e,44373,-1 for getting cluster id 2024-11-20T09:01:23,966 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T09:01:23,967 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'caf50035-b19e-4cfd-90ea-dca559866882' 2024-11-20T09:01:23,967 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T09:01:23,967 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "caf50035-b19e-4cfd-90ea-dca559866882" 2024-11-20T09:01:23,967 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@643e5114, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T09:01:23,967 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [be1eb2620e0e,44373,-1] 2024-11-20T09:01:23,967 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T09:01:23,968 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T09:01:23,968 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40598, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T09:01:23,968 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a58a201, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T09:01:23,969 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@259a56de, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T09:01:23,969 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request be1eb2620e0e,44373,-1 for getting cluster id 2024-11-20T09:01:23,969 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T09:01:23,970 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'caf50035-b19e-4cfd-90ea-dca559866882' 2024-11-20T09:01:23,970 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T09:01:23,970 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "caf50035-b19e-4cfd-90ea-dca559866882" 2024-11-20T09:01:23,970 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77f0412f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T09:01:23,970 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [be1eb2620e0e,44373,-1] 2024-11-20T09:01:23,970 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T09:01:23,970 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T09:01:23,971 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40616, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T09:01:23,971 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5cfeb4a4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T09:01:23,972 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ae789b8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T09:01:23,972 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request be1eb2620e0e,44373,-1 for getting cluster id 2024-11-20T09:01:23,972 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T09:01:23,972 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'caf50035-b19e-4cfd-90ea-dca559866882' 2024-11-20T09:01:23,972 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T09:01:23,972 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "caf50035-b19e-4cfd-90ea-dca559866882" 2024-11-20T09:01:23,973 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@342512bc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T09:01:23,973 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [be1eb2620e0e,44373,-1] 2024-11-20T09:01:23,973 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T09:01:23,973 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T09:01:23,973 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40632, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T09:01:23,974 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@183efb57, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T09:01:23,974 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b7b3b89, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T09:01:23,974 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request be1eb2620e0e,44373,-1 for getting cluster id 2024-11-20T09:01:23,974 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T09:01:23,975 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'caf50035-b19e-4cfd-90ea-dca559866882' 2024-11-20T09:01:23,975 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T09:01:23,975 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "caf50035-b19e-4cfd-90ea-dca559866882" 2024-11-20T09:01:23,975 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5928760, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T09:01:23,975 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [be1eb2620e0e,44373,-1] 2024-11-20T09:01:23,976 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T09:01:23,976 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T09:01:23,976 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40662, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T09:01:23,977 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@576bc530, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T09:01:23,977 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e04f49c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T09:01:23,977 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request be1eb2620e0e,44373,-1 for getting cluster id 2024-11-20T09:01:23,977 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T09:01:23,978 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'caf50035-b19e-4cfd-90ea-dca559866882' 2024-11-20T09:01:23,978 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T09:01:23,978 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "caf50035-b19e-4cfd-90ea-dca559866882" 2024-11-20T09:01:23,978 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3d7331a0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T09:01:23,978 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [be1eb2620e0e,44373,-1] 2024-11-20T09:01:23,978 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T09:01:23,978 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T09:01:23,979 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40680, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T09:01:23,979 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1e401dd4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T09:01:23,980 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32ca199c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T09:01:23,980 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request be1eb2620e0e,44373,-1 for getting cluster id 2024-11-20T09:01:23,980 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T09:01:23,980 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'caf50035-b19e-4cfd-90ea-dca559866882' 2024-11-20T09:01:23,981 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T09:01:23,981 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "caf50035-b19e-4cfd-90ea-dca559866882" 2024-11-20T09:01:23,981 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2537cf01, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T09:01:23,981 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [be1eb2620e0e,44373,-1] 2024-11-20T09:01:23,981 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T09:01:23,981 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T09:01:23,982 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40690, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T09:01:23,982 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a34fdd6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T09:01:23,982 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d70caa4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T09:01:23,982 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request be1eb2620e0e,44373,-1 for getting cluster id 2024-11-20T09:01:23,983 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T09:01:23,983 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'caf50035-b19e-4cfd-90ea-dca559866882' 2024-11-20T09:01:23,983 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T09:01:23,983 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "caf50035-b19e-4cfd-90ea-dca559866882" 2024-11-20T09:01:23,983 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51474508, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T09:01:23,983 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [be1eb2620e0e,44373,-1] 2024-11-20T09:01:23,983 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T09:01:23,984 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T09:01:23,984 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40716, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T09:01:23,984 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@cbe7459, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T09:01:23,985 DEBUG [Thread-2083 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T09:01:23,985 DEBUG [Thread-2096 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T09:01:23,985 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T09:01:23,985 DEBUG [Thread-2091 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T09:01:23,986 DEBUG [Thread-2087 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T09:01:23,986 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=be1eb2620e0e,34223,1732093159960, seqNum=-1] 2024-11-20T09:01:23,986 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=be1eb2620e0e,34223,1732093159960, seqNum=-1] 2024-11-20T09:01:23,986 DEBUG [Thread-2100 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T09:01:23,986 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T09:01:23,986 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T09:01:23,986 DEBUG [Thread-2098 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T09:01:23,987 DEBUG [Thread-2089 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T09:01:23,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] procedure2.ProcedureExecutor(1139): Stored pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees 2024-11-20T09:01:23,987 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=be1eb2620e0e,34223,1732093159960, seqNum=-1] 2024-11-20T09:01:23,987 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T09:01:23,987 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=be1eb2620e0e,34223,1732093159960, seqNum=-1] 2024-11-20T09:01:23,987 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=be1eb2620e0e,34223,1732093159960, seqNum=-1] 2024-11-20T09:01:23,988 DEBUG [Thread-2087 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=be1eb2620e0e,34223,1732093159960, seqNum=-1] 2024-11-20T09:01:23,988 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T09:01:23,988 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T09:01:23,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-20T09:01:23,988 DEBUG [Thread-2085 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T09:01:23,989 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T09:01:23,989 DEBUG [Thread-2102 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T09:01:23,989 DEBUG [Thread-2087 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T09:01:23,989 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=be1eb2620e0e,34223,1732093159960, seqNum=-1] 2024-11-20T09:01:23,990 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=115, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T09:01:23,990 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T09:01:23,990 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=116, ppid=115, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T09:01:23,990 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=be1eb2620e0e,34223,1732093159960, seqNum=-1] 2024-11-20T09:01:23,990 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T09:01:23,991 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52804, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T09:01:23,991 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52822, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T09:01:23,991 DEBUG [Thread-2094 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T09:01:23,991 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52814, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T09:01:23,991 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52846, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T09:01:23,992 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52832, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T09:01:23,992 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52840, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T09:01:23,992 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52854, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T09:01:23,992 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52856, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T09:01:23,992 DEBUG [Thread-2094 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=be1eb2620e0e,34223,1732093159960, seqNum=-1] 2024-11-20T09:01:23,992 DEBUG [Thread-2094 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T09:01:23,993 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2] 2024-11-20T09:01:23,993 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=be1eb2620e0e,34223,1732093159960, seqNum=-1] 2024-11-20T09:01:23,993 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T09:01:23,993 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52868, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T09:01:23,994 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2] 2024-11-20T09:01:23,994 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2] 2024-11-20T09:01:23,995 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52884, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T09:01:23,996 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2] 2024-11-20T09:01:23,996 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2] 2024-11-20T09:01:23,996 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2] 2024-11-20T09:01:23,997 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2] 2024-11-20T09:01:23,998 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2] 2024-11-20T09:01:23,999 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2] 2024-11-20T09:01:24,000 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2] 2024-11-20T09:01:24,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(8855): Flush requested on e765ef86c58446bc8e368358023e83a1 2024-11-20T09:01:24,005 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T09:01:24,005 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e765ef86c58446bc8e368358023e83a1 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-20T09:01:24,030 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/A/4bcfa11498164d259adc42674aa877aa is 50, key is test_row_0/A:col10/1732093284002/Put/seqid=0 2024-11-20T09:01:24,040 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742291_1467 (size=16681) 2024-11-20T09:01:24,068 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:24,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52846 deadline: 1732093344062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:24,068 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:24,068 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:24,068 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:01:24,077 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:24,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52884 deadline: 1732093344063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:24,078 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:24,078 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:24,078 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:24,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52840 deadline: 1732093344065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:24,078 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:01:24,078 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:24,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52854 deadline: 1732093344067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:24,078 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:24,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52804 deadline: 1732093344068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:24,079 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:24,079 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:24,079 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:01:24,079 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:24,079 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:24,079 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:01:24,079 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:24,079 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:24,079 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:01:24,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-20T09:01:24,143 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-20T09:01:24,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1. 2024-11-20T09:01:24,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1. as already flushing 2024-11-20T09:01:24,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1. 2024-11-20T09:01:24,143 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => e765ef86c58446bc8e368358023e83a1, NAME => 'TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:24,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => e765ef86c58446bc8e368358023e83a1, NAME => 'TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:24,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e765ef86c58446bc8e368358023e83a1, NAME => 'TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e765ef86c58446bc8e368358023e83a1, NAME => 'TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:24,187 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:24,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52846 deadline: 1732093344177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:24,188 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:24,188 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:24,188 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:01:24,192 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:24,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52884 deadline: 1732093344187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:24,193 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:24,193 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:24,193 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:24,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52854 deadline: 1732093344187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:24,193 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:01:24,193 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:24,193 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:24,193 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:24,193 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:01:24,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52840 deadline: 1732093344187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:24,194 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:24,194 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:24,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52804 deadline: 1732093344188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:24,194 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:24,194 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:01:24,194 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:24,194 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:24,194 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:01:24,296 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-20T09:01:24,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1. 2024-11-20T09:01:24,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1. as already flushing 2024-11-20T09:01:24,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1. 2024-11-20T09:01:24,296 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => e765ef86c58446bc8e368358023e83a1, NAME => 'TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:24,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => e765ef86c58446bc8e368358023e83a1, NAME => 'TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:24,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e765ef86c58446bc8e368358023e83a1, NAME => 'TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e765ef86c58446bc8e368358023e83a1, NAME => 'TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:24,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-20T09:01:24,406 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:24,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52846 deadline: 1732093344397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:24,407 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:24,407 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:24,407 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:24,407 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:01:24,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52884 deadline: 1732093344397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:24,407 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:24,407 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:24,407 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:24,407 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:01:24,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52854 deadline: 1732093344397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:24,407 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:24,407 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:24,407 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:24,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52840 deadline: 1732093344397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:24,407 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:01:24,408 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:24,408 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:24,408 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:01:24,408 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:24,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52804 deadline: 1732093344397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:24,408 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:24,408 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:24,408 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:01:24,441 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/A/4bcfa11498164d259adc42674aa877aa 2024-11-20T09:01:24,449 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-20T09:01:24,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1. 2024-11-20T09:01:24,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1. as already flushing 2024-11-20T09:01:24,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1. 2024-11-20T09:01:24,450 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => e765ef86c58446bc8e368358023e83a1, NAME => 'TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:24,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => e765ef86c58446bc8e368358023e83a1, NAME => 'TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:24,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e765ef86c58446bc8e368358023e83a1, NAME => 'TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e765ef86c58446bc8e368358023e83a1, NAME => 'TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:24,462 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/B/66e55a5c10844f85926e7466cc478c35 is 50, key is test_row_0/B:col10/1732093284002/Put/seqid=0 2024-11-20T09:01:24,466 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742292_1468 (size=12001) 2024-11-20T09:01:24,602 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-20T09:01:24,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1. 2024-11-20T09:01:24,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1. as already flushing 2024-11-20T09:01:24,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1. 2024-11-20T09:01:24,603 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => e765ef86c58446bc8e368358023e83a1, NAME => 'TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:24,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => e765ef86c58446bc8e368358023e83a1, NAME => 'TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:24,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e765ef86c58446bc8e368358023e83a1, NAME => 'TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e765ef86c58446bc8e368358023e83a1, NAME => 'TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:24,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-20T09:01:24,722 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:24,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52846 deadline: 1732093344717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:24,723 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:24,723 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:24,723 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:01:24,723 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:24,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52884 deadline: 1732093344717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:24,723 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:24,723 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:24,723 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:01:24,725 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:24,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52840 deadline: 1732093344717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:24,725 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:24,726 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:24,725 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:24,726 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:01:24,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52854 deadline: 1732093344717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:24,726 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:24,726 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:24,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52804 deadline: 1732093344718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:24,726 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:24,726 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:01:24,726 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:24,726 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:24,726 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:01:24,755 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-20T09:01:24,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1. 2024-11-20T09:01:24,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1. as already flushing 2024-11-20T09:01:24,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1. 2024-11-20T09:01:24,756 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => e765ef86c58446bc8e368358023e83a1, NAME => 'TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:24,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => e765ef86c58446bc8e368358023e83a1, NAME => 'TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:24,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e765ef86c58446bc8e368358023e83a1, NAME => 'TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e765ef86c58446bc8e368358023e83a1, NAME => 'TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:24,867 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/B/66e55a5c10844f85926e7466cc478c35 2024-11-20T09:01:24,890 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/C/cb5737d0e45c4fd291bfaa472ea1094f is 50, key is test_row_0/C:col10/1732093284002/Put/seqid=0 2024-11-20T09:01:24,894 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742293_1469 (size=12001) 2024-11-20T09:01:24,908 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-20T09:01:24,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1. 2024-11-20T09:01:24,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1. as already flushing 2024-11-20T09:01:24,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1. 2024-11-20T09:01:24,909 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => e765ef86c58446bc8e368358023e83a1, NAME => 'TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:24,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => e765ef86c58446bc8e368358023e83a1, NAME => 'TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:24,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e765ef86c58446bc8e368358023e83a1, NAME => 'TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e765ef86c58446bc8e368358023e83a1, NAME => 'TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:25,061 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-20T09:01:25,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1. 2024-11-20T09:01:25,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1. as already flushing 2024-11-20T09:01:25,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1. 2024-11-20T09:01:25,062 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => e765ef86c58446bc8e368358023e83a1, NAME => 'TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:25,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => e765ef86c58446bc8e368358023e83a1, NAME => 'TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:25,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e765ef86c58446bc8e368358023e83a1, NAME => 'TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e765ef86c58446bc8e368358023e83a1, NAME => 'TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:25,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-20T09:01:25,213 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-20T09:01:25,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1. 2024-11-20T09:01:25,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1. as already flushing 2024-11-20T09:01:25,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1. 2024-11-20T09:01:25,214 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => e765ef86c58446bc8e368358023e83a1, NAME => 'TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:25,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => e765ef86c58446bc8e368358023e83a1, NAME => 'TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:25,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e765ef86c58446bc8e368358023e83a1, NAME => 'TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e765ef86c58446bc8e368358023e83a1, NAME => 'TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:25,238 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:25,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52884 deadline: 1732093345227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:25,239 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:25,239 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:25,239 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:01:25,246 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:25,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52846 deadline: 1732093345237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:25,246 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:25,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52840 deadline: 1732093345238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:25,246 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:25,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52854 deadline: 1732093345238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:25,247 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:25,247 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:25,247 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:01:25,247 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:25,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52804 deadline: 1732093345238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:25,247 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:25,247 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:25,247 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:01:25,247 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:25,247 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:25,247 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:01:25,248 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:25,248 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:25,248 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:01:25,295 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/C/cb5737d0e45c4fd291bfaa472ea1094f 2024-11-20T09:01:25,299 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/A/4bcfa11498164d259adc42674aa877aa as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/A/4bcfa11498164d259adc42674aa877aa 2024-11-20T09:01:25,302 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/A/4bcfa11498164d259adc42674aa877aa, entries=250, sequenceid=13, filesize=16.3 K 2024-11-20T09:01:25,303 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/B/66e55a5c10844f85926e7466cc478c35 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/B/66e55a5c10844f85926e7466cc478c35 2024-11-20T09:01:25,306 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/B/66e55a5c10844f85926e7466cc478c35, entries=150, sequenceid=13, filesize=11.7 K 2024-11-20T09:01:25,307 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/C/cb5737d0e45c4fd291bfaa472ea1094f as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/C/cb5737d0e45c4fd291bfaa472ea1094f 2024-11-20T09:01:25,310 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/C/cb5737d0e45c4fd291bfaa472ea1094f, entries=150, sequenceid=13, filesize=11.7 K 2024-11-20T09:01:25,311 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for e765ef86c58446bc8e368358023e83a1 in 1306ms, sequenceid=13, compaction requested=false 2024-11-20T09:01:25,311 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-20T09:01:25,311 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e765ef86c58446bc8e368358023e83a1: 2024-11-20T09:01:25,367 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-20T09:01:25,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1. 2024-11-20T09:01:25,368 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2902): Flushing e765ef86c58446bc8e368358023e83a1 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-20T09:01:25,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/A/e8defb143a7f4be3be7b3cff158d70c4 is 50, key is test_row_0/A:col10/1732093284063/Put/seqid=0 2024-11-20T09:01:25,381 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742294_1470 (size=12001) 2024-11-20T09:01:25,782 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/A/e8defb143a7f4be3be7b3cff158d70c4 2024-11-20T09:01:25,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/B/64af32b0fdd3480fb334d6c518434b67 is 50, key is test_row_0/B:col10/1732093284063/Put/seqid=0 2024-11-20T09:01:25,809 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742295_1471 (size=12001) 2024-11-20T09:01:25,809 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/B/64af32b0fdd3480fb334d6c518434b67 2024-11-20T09:01:25,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/C/a1de1814bc8d4ca7bca9174a70789e78 is 50, key is test_row_0/C:col10/1732093284063/Put/seqid=0 2024-11-20T09:01:25,840 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742296_1472 (size=12001) 2024-11-20T09:01:26,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-20T09:01:26,235 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/C/a1de1814bc8d4ca7bca9174a70789e78 2024-11-20T09:01:26,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/A/e8defb143a7f4be3be7b3cff158d70c4 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/A/e8defb143a7f4be3be7b3cff158d70c4 2024-11-20T09:01:26,243 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/A/e8defb143a7f4be3be7b3cff158d70c4, entries=150, sequenceid=37, filesize=11.7 K 2024-11-20T09:01:26,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/B/64af32b0fdd3480fb334d6c518434b67 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/B/64af32b0fdd3480fb334d6c518434b67 2024-11-20T09:01:26,247 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/B/64af32b0fdd3480fb334d6c518434b67, entries=150, sequenceid=37, filesize=11.7 K 2024-11-20T09:01:26,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/C/a1de1814bc8d4ca7bca9174a70789e78 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/C/a1de1814bc8d4ca7bca9174a70789e78 2024-11-20T09:01:26,252 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/C/a1de1814bc8d4ca7bca9174a70789e78, entries=150, sequenceid=37, filesize=11.7 K 2024-11-20T09:01:26,253 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=0 B/0 for e765ef86c58446bc8e368358023e83a1 in 886ms, sequenceid=37, compaction requested=false 2024-11-20T09:01:26,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2603): Flush status journal for e765ef86c58446bc8e368358023e83a1: 2024-11-20T09:01:26,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1. 2024-11-20T09:01:26,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=116 2024-11-20T09:01:26,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.HMaster(4169): Remote procedure done, pid=116 2024-11-20T09:01:26,256 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=116, resume processing ppid=115 2024-11-20T09:01:26,256 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=116, ppid=115, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2640 sec 2024-11-20T09:01:26,259 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=115, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees in 2.2720 sec 2024-11-20T09:01:26,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(8855): Flush requested on e765ef86c58446bc8e368358023e83a1 2024-11-20T09:01:26,266 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T09:01:26,266 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e765ef86c58446bc8e368358023e83a1 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-20T09:01:26,273 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/A/3ffc2f0d13a343048bbbf9e77d1d95ce is 50, key is test_row_0/A:col10/1732093286257/Put/seqid=0 2024-11-20T09:01:26,276 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742297_1473 (size=19021) 2024-11-20T09:01:26,277 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/A/3ffc2f0d13a343048bbbf9e77d1d95ce 2024-11-20T09:01:26,284 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/B/79c5fe4a80fc4d31bc261f8494f38eae is 50, key is test_row_0/B:col10/1732093286257/Put/seqid=0 2024-11-20T09:01:26,293 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742298_1474 (size=12001) 2024-11-20T09:01:26,312 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:26,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52854 deadline: 1732093346300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:26,312 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:26,313 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:26,313 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:01:26,313 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:26,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52846 deadline: 1732093346301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:26,313 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:26,313 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:26,313 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:26,313 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:01:26,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52840 deadline: 1732093346302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:26,313 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:26,313 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:26,313 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:26,313 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:01:26,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52804 deadline: 1732093346311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:26,314 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:26,314 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:26,314 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:01:26,327 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:26,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52884 deadline: 1732093346312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:26,327 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:26,327 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:26,327 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:01:26,423 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:26,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52854 deadline: 1732093346416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:26,424 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:26,424 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:26,424 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:26,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52846 deadline: 1732093346417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:26,424 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:01:26,424 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:26,424 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:26,424 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:01:26,426 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:26,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52804 deadline: 1732093346417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:26,427 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:26,427 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:26,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52840 deadline: 1732093346417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:26,427 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:26,427 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:01:26,427 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:26,427 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:26,427 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:01:26,439 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:26,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52884 deadline: 1732093346437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:26,439 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:26,439 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:26,439 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:01:26,632 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:26,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52854 deadline: 1732093346627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:26,633 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:26,633 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:26,633 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:26,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52846 deadline: 1732093346627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:26,633 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:01:26,633 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:26,633 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:26,633 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:01:26,645 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:26,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52804 deadline: 1732093346637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:26,645 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:26,645 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:26,645 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:26,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52840 deadline: 1732093346637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:26,645 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:01:26,646 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:26,646 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:26,646 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:01:26,657 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:26,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52884 deadline: 1732093346646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:26,658 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:26,658 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:26,658 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:01:26,694 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/B/79c5fe4a80fc4d31bc261f8494f38eae 2024-11-20T09:01:26,703 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/C/bf8037924a7a4c718b05880d914f422e is 50, key is test_row_0/C:col10/1732093286257/Put/seqid=0 2024-11-20T09:01:26,711 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742299_1475 (size=12001) 2024-11-20T09:01:26,712 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/C/bf8037924a7a4c718b05880d914f422e 2024-11-20T09:01:26,715 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/A/3ffc2f0d13a343048bbbf9e77d1d95ce as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/A/3ffc2f0d13a343048bbbf9e77d1d95ce 2024-11-20T09:01:26,718 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/A/3ffc2f0d13a343048bbbf9e77d1d95ce, entries=300, sequenceid=49, filesize=18.6 K 2024-11-20T09:01:26,719 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/B/79c5fe4a80fc4d31bc261f8494f38eae as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/B/79c5fe4a80fc4d31bc261f8494f38eae 2024-11-20T09:01:26,728 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/B/79c5fe4a80fc4d31bc261f8494f38eae, entries=150, sequenceid=49, filesize=11.7 K 2024-11-20T09:01:26,729 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/C/bf8037924a7a4c718b05880d914f422e as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/C/bf8037924a7a4c718b05880d914f422e 2024-11-20T09:01:26,733 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/C/bf8037924a7a4c718b05880d914f422e, entries=150, sequenceid=49, filesize=11.7 K 2024-11-20T09:01:26,734 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for e765ef86c58446bc8e368358023e83a1 in 468ms, sequenceid=49, compaction requested=true 2024-11-20T09:01:26,734 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e765ef86c58446bc8e368358023e83a1: 2024-11-20T09:01:26,734 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e765ef86c58446bc8e368358023e83a1:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T09:01:26,734 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T09:01:26,735 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T09:01:26,735 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T09:01:26,735 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e765ef86c58446bc8e368358023e83a1:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T09:01:26,735 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T09:01:26,735 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e765ef86c58446bc8e368358023e83a1:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T09:01:26,735 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T09:01:26,737 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 47703 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T09:01:26,737 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HStore(1541): e765ef86c58446bc8e368358023e83a1/A is initiating minor compaction (all files) 2024-11-20T09:01:26,737 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e765ef86c58446bc8e368358023e83a1/A in TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1. 2024-11-20T09:01:26,737 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/A/4bcfa11498164d259adc42674aa877aa, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/A/e8defb143a7f4be3be7b3cff158d70c4, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/A/3ffc2f0d13a343048bbbf9e77d1d95ce] into tmpdir=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp, totalSize=46.6 K 2024-11-20T09:01:26,738 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T09:01:26,738 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4bcfa11498164d259adc42674aa877aa, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1732093283998 2024-11-20T09:01:26,738 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1541): e765ef86c58446bc8e368358023e83a1/B is initiating minor compaction (all files) 2024-11-20T09:01:26,738 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e765ef86c58446bc8e368358023e83a1/B in TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1. 2024-11-20T09:01:26,738 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/B/66e55a5c10844f85926e7466cc478c35, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/B/64af32b0fdd3480fb334d6c518434b67, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/B/79c5fe4a80fc4d31bc261f8494f38eae] into tmpdir=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp, totalSize=35.2 K 2024-11-20T09:01:26,738 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.Compactor(225): Compacting e8defb143a7f4be3be7b3cff158d70c4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1732093284063 2024-11-20T09:01:26,738 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting 66e55a5c10844f85926e7466cc478c35, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1732093283999 2024-11-20T09:01:26,739 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3ffc2f0d13a343048bbbf9e77d1d95ce, keycount=300, bloomtype=ROW, size=18.6 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732093286257 2024-11-20T09:01:26,739 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting 64af32b0fdd3480fb334d6c518434b67, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1732093284063 2024-11-20T09:01:26,739 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting 79c5fe4a80fc4d31bc261f8494f38eae, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732093286257 2024-11-20T09:01:26,747 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e765ef86c58446bc8e368358023e83a1#A#compaction#398 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T09:01:26,748 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/A/77908ad062e74187b8bf9466aef85d48 is 50, key is test_row_0/A:col10/1732093286257/Put/seqid=0 2024-11-20T09:01:26,754 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e765ef86c58446bc8e368358023e83a1#B#compaction#399 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T09:01:26,754 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/B/697b5306184b4b1e8541e75ab8f7670a is 50, key is test_row_0/B:col10/1732093286257/Put/seqid=0 2024-11-20T09:01:26,770 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-20T09:01:26,787 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742300_1476 (size=12104) 2024-11-20T09:01:26,796 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742301_1477 (size=12104) 2024-11-20T09:01:26,802 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/A/77908ad062e74187b8bf9466aef85d48 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/A/77908ad062e74187b8bf9466aef85d48 2024-11-20T09:01:26,809 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e765ef86c58446bc8e368358023e83a1/A of e765ef86c58446bc8e368358023e83a1 into 77908ad062e74187b8bf9466aef85d48(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T09:01:26,809 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e765ef86c58446bc8e368358023e83a1: 2024-11-20T09:01:26,809 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., storeName=e765ef86c58446bc8e368358023e83a1/A, priority=13, startTime=1732093286734; duration=0sec 2024-11-20T09:01:26,809 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T09:01:26,809 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e765ef86c58446bc8e368358023e83a1:A 2024-11-20T09:01:26,809 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T09:01:26,813 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T09:01:26,813 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HStore(1541): e765ef86c58446bc8e368358023e83a1/C is initiating minor compaction (all files) 2024-11-20T09:01:26,813 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e765ef86c58446bc8e368358023e83a1/C in TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1. 2024-11-20T09:01:26,813 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/C/cb5737d0e45c4fd291bfaa472ea1094f, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/C/a1de1814bc8d4ca7bca9174a70789e78, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/C/bf8037924a7a4c718b05880d914f422e] into tmpdir=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp, totalSize=35.2 K 2024-11-20T09:01:26,813 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/B/697b5306184b4b1e8541e75ab8f7670a as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/B/697b5306184b4b1e8541e75ab8f7670a 2024-11-20T09:01:26,814 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.Compactor(225): Compacting cb5737d0e45c4fd291bfaa472ea1094f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1732093283999 2024-11-20T09:01:26,814 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.Compactor(225): Compacting a1de1814bc8d4ca7bca9174a70789e78, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1732093284063 2024-11-20T09:01:26,814 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.Compactor(225): Compacting bf8037924a7a4c718b05880d914f422e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732093286257 2024-11-20T09:01:26,819 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e765ef86c58446bc8e368358023e83a1/B of e765ef86c58446bc8e368358023e83a1 into 697b5306184b4b1e8541e75ab8f7670a(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T09:01:26,819 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e765ef86c58446bc8e368358023e83a1: 2024-11-20T09:01:26,819 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., storeName=e765ef86c58446bc8e368358023e83a1/B, priority=13, startTime=1732093286735; duration=0sec 2024-11-20T09:01:26,819 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T09:01:26,819 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e765ef86c58446bc8e368358023e83a1:B 2024-11-20T09:01:26,830 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e765ef86c58446bc8e368358023e83a1#C#compaction#400 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T09:01:26,830 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/C/a3d7531c88fd4c1088abd241acd76c64 is 50, key is test_row_0/C:col10/1732093286257/Put/seqid=0 2024-11-20T09:01:26,842 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742302_1478 (size=12104) 2024-11-20T09:01:26,860 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/C/a3d7531c88fd4c1088abd241acd76c64 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/C/a3d7531c88fd4c1088abd241acd76c64 2024-11-20T09:01:26,870 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e765ef86c58446bc8e368358023e83a1/C of e765ef86c58446bc8e368358023e83a1 into a3d7531c88fd4c1088abd241acd76c64(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T09:01:26,870 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e765ef86c58446bc8e368358023e83a1: 2024-11-20T09:01:26,870 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., storeName=e765ef86c58446bc8e368358023e83a1/C, priority=13, startTime=1732093286735; duration=0sec 2024-11-20T09:01:26,871 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T09:01:26,871 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e765ef86c58446bc8e368358023e83a1:C 2024-11-20T09:01:26,944 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T09:01:26,944 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e765ef86c58446bc8e368358023e83a1 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-20T09:01:26,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(8855): Flush requested on e765ef86c58446bc8e368358023e83a1 2024-11-20T09:01:26,950 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/A/6de8a38311a44bc88291f16ca0a48e3b is 50, key is test_row_0/A:col10/1732093286942/Put/seqid=0 2024-11-20T09:01:26,964 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742303_1479 (size=14341) 2024-11-20T09:01:26,974 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:26,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52846 deadline: 1732093346965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:26,975 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:26,975 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:26,975 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:26,975 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:01:26,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52804 deadline: 1732093346966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:26,975 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:26,975 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:26,975 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:26,975 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:01:26,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52884 deadline: 1732093346968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:26,976 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:26,976 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:26,976 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:01:26,984 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:26,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52854 deadline: 1732093346971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:26,984 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:26,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52840 deadline: 1732093346971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:26,985 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:26,985 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:26,985 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:01:26,986 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:26,986 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:26,986 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:01:27,081 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:27,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52846 deadline: 1732093347077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:27,082 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:27,082 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:27,082 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:01:27,091 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:27,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52804 deadline: 1732093347087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:27,092 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:27,092 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:27,092 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:27,092 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:01:27,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52840 deadline: 1732093347087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:27,094 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:27,094 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:27,094 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:01:27,104 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:27,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52854 deadline: 1732093347096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:27,105 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:27,105 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:27,105 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:01:27,290 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:27,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52846 deadline: 1732093347287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:27,290 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:27,290 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:27,290 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:01:27,299 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:27,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52804 deadline: 1732093347297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:27,300 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:27,300 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:27,300 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:01:27,300 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:27,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52840 deadline: 1732093347298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:27,300 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:27,300 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:27,300 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:01:27,325 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:27,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52854 deadline: 1732093347321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:27,328 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:27,328 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:27,328 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:01:27,366 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/A/6de8a38311a44bc88291f16ca0a48e3b 2024-11-20T09:01:27,374 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/B/6e49b35edd094ed880e578eb9ed9a94d is 50, key is test_row_0/B:col10/1732093286942/Put/seqid=0 2024-11-20T09:01:27,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742304_1480 (size=12001) 2024-11-20T09:01:27,398 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/B/6e49b35edd094ed880e578eb9ed9a94d 2024-11-20T09:01:27,409 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/C/e646e522f7b34fc298e8c4571d11232d is 50, key is test_row_0/C:col10/1732093286942/Put/seqid=0 2024-11-20T09:01:27,423 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742305_1481 (size=12001) 2024-11-20T09:01:27,495 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:27,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52884 deadline: 1732093347487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:27,495 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:27,496 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:27,496 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:01:27,605 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:27,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52846 deadline: 1732093347597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:27,605 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:27,606 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:27,606 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:01:27,611 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:27,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52804 deadline: 1732093347606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:27,612 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:27,612 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:27,612 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:27,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52840 deadline: 1732093347606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:27,612 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:01:27,612 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:27,612 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:27,612 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:01:27,644 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:27,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52854 deadline: 1732093347637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:27,645 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:27,645 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:27,645 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:01:27,824 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/C/e646e522f7b34fc298e8c4571d11232d 2024-11-20T09:01:27,828 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/A/6de8a38311a44bc88291f16ca0a48e3b as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/A/6de8a38311a44bc88291f16ca0a48e3b 2024-11-20T09:01:27,832 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/A/6de8a38311a44bc88291f16ca0a48e3b, entries=200, sequenceid=77, filesize=14.0 K 2024-11-20T09:01:27,832 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/B/6e49b35edd094ed880e578eb9ed9a94d as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/B/6e49b35edd094ed880e578eb9ed9a94d 2024-11-20T09:01:27,835 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/B/6e49b35edd094ed880e578eb9ed9a94d, entries=150, sequenceid=77, filesize=11.7 K 2024-11-20T09:01:27,836 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/C/e646e522f7b34fc298e8c4571d11232d as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/C/e646e522f7b34fc298e8c4571d11232d 2024-11-20T09:01:27,840 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/C/e646e522f7b34fc298e8c4571d11232d, entries=150, sequenceid=77, filesize=11.7 K 2024-11-20T09:01:27,840 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for e765ef86c58446bc8e368358023e83a1 in 896ms, sequenceid=77, compaction requested=false 2024-11-20T09:01:27,840 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e765ef86c58446bc8e368358023e83a1: 2024-11-20T09:01:28,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(8855): Flush requested on e765ef86c58446bc8e368358023e83a1 2024-11-20T09:01:28,119 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T09:01:28,119 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e765ef86c58446bc8e368358023e83a1 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-20T09:01:28,124 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/A/44449f6c51fb4e0e9f63c4358bede375 is 50, key is test_row_0/A:col10/1732093286968/Put/seqid=0 2024-11-20T09:01:28,128 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742306_1482 (size=14341) 2024-11-20T09:01:28,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-20T09:01:28,147 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T09:01:28,147 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T09:01:28,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] procedure2.ProcedureExecutor(1139): Stored pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees 2024-11-20T09:01:28,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-20T09:01:28,149 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T09:01:28,150 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T09:01:28,150 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=118, ppid=117, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T09:01:28,184 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:28,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52804 deadline: 1732093348170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:28,184 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:28,184 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:28,184 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:28,184 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:01:28,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52840 deadline: 1732093348173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:28,185 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:28,185 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:28,185 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:01:28,195 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:28,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52854 deadline: 1732093348183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:28,196 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:28,196 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:28,196 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:01:28,199 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:28,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52846 deadline: 1732093348184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:28,199 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:28,199 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:28,199 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:01:28,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-20T09:01:28,296 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:28,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52804 deadline: 1732093348287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:28,296 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:28,296 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:28,296 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:01:28,296 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:28,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52840 deadline: 1732093348287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:28,297 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:28,297 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:28,297 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:01:28,304 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-20T09:01:28,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1. 2024-11-20T09:01:28,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1. as already flushing 2024-11-20T09:01:28,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1. 2024-11-20T09:01:28,305 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => e765ef86c58446bc8e368358023e83a1, NAME => 'TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:28,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => e765ef86c58446bc8e368358023e83a1, NAME => 'TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:28,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e765ef86c58446bc8e368358023e83a1, NAME => 'TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e765ef86c58446bc8e368358023e83a1, NAME => 'TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:28,314 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:28,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52854 deadline: 1732093348307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:28,314 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:28,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52846 deadline: 1732093348308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:28,314 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:28,314 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:28,314 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:01:28,314 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:28,314 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:28,314 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:01:28,459 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-20T09:01:28,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1. 2024-11-20T09:01:28,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1. as already flushing 2024-11-20T09:01:28,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1. 2024-11-20T09:01:28,459 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => e765ef86c58446bc8e368358023e83a1, NAME => 'TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:28,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => e765ef86c58446bc8e368358023e83a1, NAME => 'TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:28,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e765ef86c58446bc8e368358023e83a1, NAME => 'TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e765ef86c58446bc8e368358023e83a1, NAME => 'TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:28,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-20T09:01:28,511 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:28,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52884 deadline: 1732093348507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:28,511 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:28,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52804 deadline: 1732093348507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:28,512 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:28,512 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:28,512 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:28,512 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:01:28,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52840 deadline: 1732093348507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:28,511 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to be1eb2620e0e,34223,1732093159960 for 'test_row_1' in e765ef86c58446bc8e368358023e83a1 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2199 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-20T09:01:28,512 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:28,512 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:28,512 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:28,512 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:28,512 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:01:28,512 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:01:28,523 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:28,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52854 deadline: 1732093348517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:28,524 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:28,524 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:28,524 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:28,524 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:01:28,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52846 deadline: 1732093348517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:28,524 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:28,524 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:28,524 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:01:28,528 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=89 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/A/44449f6c51fb4e0e9f63c4358bede375 2024-11-20T09:01:28,535 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/B/d908eb4390cd4940ba7b5fe463c91ba1 is 50, key is test_row_0/B:col10/1732093286968/Put/seqid=0 2024-11-20T09:01:28,539 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742307_1483 (size=12001) 2024-11-20T09:01:28,613 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-20T09:01:28,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1. 2024-11-20T09:01:28,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1. as already flushing 2024-11-20T09:01:28,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1. 2024-11-20T09:01:28,614 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => e765ef86c58446bc8e368358023e83a1, NAME => 'TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:28,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => e765ef86c58446bc8e368358023e83a1, NAME => 'TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:28,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e765ef86c58446bc8e368358023e83a1, NAME => 'TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e765ef86c58446bc8e368358023e83a1, NAME => 'TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:28,766 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-20T09:01:28,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1. 2024-11-20T09:01:28,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1. as already flushing 2024-11-20T09:01:28,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1. 2024-11-20T09:01:28,767 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => e765ef86c58446bc8e368358023e83a1, NAME => 'TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:28,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => e765ef86c58446bc8e368358023e83a1, NAME => 'TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:28,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e765ef86c58446bc8e368358023e83a1, NAME => 'TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e765ef86c58446bc8e368358023e83a1, NAME => 'TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:28,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-20T09:01:28,818 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:28,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52804 deadline: 1732093348816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:28,818 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:28,819 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:28,819 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:28,819 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:01:28,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52840 deadline: 1732093348816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:28,819 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:28,819 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:28,819 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:01:28,831 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:28,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52854 deadline: 1732093348827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:28,831 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:28,831 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:28,831 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:01:28,843 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:28,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52846 deadline: 1732093348837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:28,844 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:28,844 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:28,844 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:01:28,920 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-20T09:01:28,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1. 2024-11-20T09:01:28,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1. as already flushing 2024-11-20T09:01:28,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1. 2024-11-20T09:01:28,920 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => e765ef86c58446bc8e368358023e83a1, NAME => 'TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:28,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => e765ef86c58446bc8e368358023e83a1, NAME => 'TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:28,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e765ef86c58446bc8e368358023e83a1, NAME => 'TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e765ef86c58446bc8e368358023e83a1, NAME => 'TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:28,940 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=89 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/B/d908eb4390cd4940ba7b5fe463c91ba1 2024-11-20T09:01:28,947 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/C/eb53dbb857d941fc97c2e076e36b67ca is 50, key is test_row_0/C:col10/1732093286968/Put/seqid=0 2024-11-20T09:01:28,950 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742308_1484 (size=12001) 2024-11-20T09:01:28,951 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=89 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/C/eb53dbb857d941fc97c2e076e36b67ca 2024-11-20T09:01:28,954 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/A/44449f6c51fb4e0e9f63c4358bede375 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/A/44449f6c51fb4e0e9f63c4358bede375 2024-11-20T09:01:28,956 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/A/44449f6c51fb4e0e9f63c4358bede375, entries=200, sequenceid=89, filesize=14.0 K 2024-11-20T09:01:28,957 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/B/d908eb4390cd4940ba7b5fe463c91ba1 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/B/d908eb4390cd4940ba7b5fe463c91ba1 2024-11-20T09:01:28,960 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/B/d908eb4390cd4940ba7b5fe463c91ba1, entries=150, sequenceid=89, filesize=11.7 K 2024-11-20T09:01:28,961 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/C/eb53dbb857d941fc97c2e076e36b67ca as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/C/eb53dbb857d941fc97c2e076e36b67ca 2024-11-20T09:01:28,965 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/C/eb53dbb857d941fc97c2e076e36b67ca, entries=150, sequenceid=89, filesize=11.7 K 2024-11-20T09:01:28,966 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for e765ef86c58446bc8e368358023e83a1 in 847ms, sequenceid=89, compaction requested=true 2024-11-20T09:01:28,966 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e765ef86c58446bc8e368358023e83a1: 2024-11-20T09:01:28,966 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e765ef86c58446bc8e368358023e83a1:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T09:01:28,966 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T09:01:28,966 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e765ef86c58446bc8e368358023e83a1:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T09:01:28,966 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T09:01:28,966 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T09:01:28,966 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e765ef86c58446bc8e368358023e83a1:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T09:01:28,966 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T09:01:28,966 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T09:01:28,967 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40786 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T09:01:28,967 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T09:01:28,967 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HStore(1541): e765ef86c58446bc8e368358023e83a1/A is initiating minor compaction (all files) 2024-11-20T09:01:28,967 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1541): e765ef86c58446bc8e368358023e83a1/B is initiating minor compaction (all files) 2024-11-20T09:01:28,967 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e765ef86c58446bc8e368358023e83a1/A in TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1. 2024-11-20T09:01:28,967 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e765ef86c58446bc8e368358023e83a1/B in TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1. 2024-11-20T09:01:28,967 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/A/77908ad062e74187b8bf9466aef85d48, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/A/6de8a38311a44bc88291f16ca0a48e3b, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/A/44449f6c51fb4e0e9f63c4358bede375] into tmpdir=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp, totalSize=39.8 K 2024-11-20T09:01:28,967 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/B/697b5306184b4b1e8541e75ab8f7670a, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/B/6e49b35edd094ed880e578eb9ed9a94d, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/B/d908eb4390cd4940ba7b5fe463c91ba1] into tmpdir=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp, totalSize=35.3 K 2024-11-20T09:01:28,967 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting 697b5306184b4b1e8541e75ab8f7670a, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732093286257 2024-11-20T09:01:28,967 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.Compactor(225): Compacting 77908ad062e74187b8bf9466aef85d48, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732093286257 2024-11-20T09:01:28,968 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6de8a38311a44bc88291f16ca0a48e3b, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1732093286300 2024-11-20T09:01:28,968 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting 6e49b35edd094ed880e578eb9ed9a94d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1732093286300 2024-11-20T09:01:28,968 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.Compactor(225): Compacting 44449f6c51fb4e0e9f63c4358bede375, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1732093286954 2024-11-20T09:01:28,968 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting d908eb4390cd4940ba7b5fe463c91ba1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1732093286962 2024-11-20T09:01:28,983 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e765ef86c58446bc8e368358023e83a1#A#compaction#407 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-20T09:01:28,983 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e765ef86c58446bc8e368358023e83a1#B#compaction#408 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T09:01:28,983 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/A/56fb634a8cd646e0987e34ed4d457fce is 50, key is test_row_0/A:col10/1732093286968/Put/seqid=0 2024-11-20T09:01:28,983 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/B/dcb8ea1cb8f34f3892bde30386b04ef2 is 50, key is test_row_0/B:col10/1732093286968/Put/seqid=0 2024-11-20T09:01:28,997 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742309_1485 (size=12207) 2024-11-20T09:01:28,998 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742310_1486 (size=12207) 2024-11-20T09:01:29,002 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/A/56fb634a8cd646e0987e34ed4d457fce as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/A/56fb634a8cd646e0987e34ed4d457fce 2024-11-20T09:01:29,009 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e765ef86c58446bc8e368358023e83a1/A of e765ef86c58446bc8e368358023e83a1 into 56fb634a8cd646e0987e34ed4d457fce(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T09:01:29,009 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e765ef86c58446bc8e368358023e83a1: 2024-11-20T09:01:29,009 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., storeName=e765ef86c58446bc8e368358023e83a1/A, priority=13, startTime=1732093288966; duration=0sec 2024-11-20T09:01:29,009 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T09:01:29,009 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e765ef86c58446bc8e368358023e83a1:A 2024-11-20T09:01:29,009 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T09:01:29,010 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T09:01:29,010 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HStore(1541): e765ef86c58446bc8e368358023e83a1/C is initiating minor compaction (all files) 2024-11-20T09:01:29,010 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e765ef86c58446bc8e368358023e83a1/C in TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1. 2024-11-20T09:01:29,011 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/C/a3d7531c88fd4c1088abd241acd76c64, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/C/e646e522f7b34fc298e8c4571d11232d, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/C/eb53dbb857d941fc97c2e076e36b67ca] into tmpdir=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp, totalSize=35.3 K 2024-11-20T09:01:29,011 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.Compactor(225): Compacting a3d7531c88fd4c1088abd241acd76c64, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732093286257 2024-11-20T09:01:29,011 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/B/dcb8ea1cb8f34f3892bde30386b04ef2 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/B/dcb8ea1cb8f34f3892bde30386b04ef2 2024-11-20T09:01:29,011 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.Compactor(225): Compacting e646e522f7b34fc298e8c4571d11232d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1732093286300 2024-11-20T09:01:29,012 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.Compactor(225): Compacting eb53dbb857d941fc97c2e076e36b67ca, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1732093286962 2024-11-20T09:01:29,018 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e765ef86c58446bc8e368358023e83a1/B of e765ef86c58446bc8e368358023e83a1 into dcb8ea1cb8f34f3892bde30386b04ef2(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T09:01:29,018 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e765ef86c58446bc8e368358023e83a1: 2024-11-20T09:01:29,018 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., storeName=e765ef86c58446bc8e368358023e83a1/B, priority=13, startTime=1732093288966; duration=0sec 2024-11-20T09:01:29,018 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T09:01:29,018 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e765ef86c58446bc8e368358023e83a1:B 2024-11-20T09:01:29,025 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e765ef86c58446bc8e368358023e83a1#C#compaction#409 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T09:01:29,026 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/C/b606d8d0977e41758458c67f49385797 is 50, key is test_row_0/C:col10/1732093286968/Put/seqid=0 2024-11-20T09:01:29,029 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742311_1487 (size=12207) 2024-11-20T09:01:29,073 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-20T09:01:29,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1. 2024-11-20T09:01:29,073 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2902): Flushing e765ef86c58446bc8e368358023e83a1 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-20T09:01:29,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/A/7930cf28cb00469591a56b01b26e5f3a is 50, key is test_row_0/A:col10/1732093288182/Put/seqid=0 2024-11-20T09:01:29,083 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742312_1488 (size=12001) 2024-11-20T09:01:29,084 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/A/7930cf28cb00469591a56b01b26e5f3a 2024-11-20T09:01:29,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/B/9d9902e4ddea40a4888ab266b0103afb is 50, key is test_row_0/B:col10/1732093288182/Put/seqid=0 2024-11-20T09:01:29,095 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742313_1489 (size=12001) 2024-11-20T09:01:29,096 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/B/9d9902e4ddea40a4888ab266b0103afb 2024-11-20T09:01:29,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/C/f9860385e33d4af6b0840ef0c0650bc8 is 50, key is test_row_0/C:col10/1732093288182/Put/seqid=0 2024-11-20T09:01:29,109 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742314_1490 (size=12001) 2024-11-20T09:01:29,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-20T09:01:29,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(8855): Flush requested on e765ef86c58446bc8e368358023e83a1 2024-11-20T09:01:29,331 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1. as already flushing 2024-11-20T09:01:29,360 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:29,360 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:29,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52840 deadline: 1732093349346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:29,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52846 deadline: 1732093349347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:29,360 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:29,360 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:29,360 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:29,360 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:29,360 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:29,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52804 deadline: 1732093349348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:29,360 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:01:29,360 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:01:29,360 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:29,361 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:29,360 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:29,361 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:01:29,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52854 deadline: 1732093349355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:29,361 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:29,361 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:29,361 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:01:29,433 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/C/b606d8d0977e41758458c67f49385797 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/C/b606d8d0977e41758458c67f49385797 2024-11-20T09:01:29,437 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e765ef86c58446bc8e368358023e83a1/C of e765ef86c58446bc8e368358023e83a1 into b606d8d0977e41758458c67f49385797(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T09:01:29,437 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e765ef86c58446bc8e368358023e83a1: 2024-11-20T09:01:29,437 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., storeName=e765ef86c58446bc8e368358023e83a1/C, priority=13, startTime=1732093288966; duration=0sec 2024-11-20T09:01:29,437 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T09:01:29,437 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e765ef86c58446bc8e368358023e83a1:C 2024-11-20T09:01:29,474 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:29,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52840 deadline: 1732093349467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:29,475 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:29,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52804 deadline: 1732093349467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:29,475 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:29,475 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:29,475 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:01:29,475 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:29,475 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:29,475 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:29,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52854 deadline: 1732093349467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:29,475 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:01:29,475 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:29,475 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:29,475 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:01:29,510 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/C/f9860385e33d4af6b0840ef0c0650bc8 2024-11-20T09:01:29,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/A/7930cf28cb00469591a56b01b26e5f3a as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/A/7930cf28cb00469591a56b01b26e5f3a 2024-11-20T09:01:29,516 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/A/7930cf28cb00469591a56b01b26e5f3a, entries=150, sequenceid=115, filesize=11.7 K 2024-11-20T09:01:29,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/B/9d9902e4ddea40a4888ab266b0103afb as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/B/9d9902e4ddea40a4888ab266b0103afb 2024-11-20T09:01:29,521 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/B/9d9902e4ddea40a4888ab266b0103afb, entries=150, sequenceid=115, filesize=11.7 K 2024-11-20T09:01:29,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/C/f9860385e33d4af6b0840ef0c0650bc8 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/C/f9860385e33d4af6b0840ef0c0650bc8 2024-11-20T09:01:29,525 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/C/f9860385e33d4af6b0840ef0c0650bc8, entries=150, sequenceid=115, filesize=11.7 K 2024-11-20T09:01:29,526 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for e765ef86c58446bc8e368358023e83a1 in 453ms, sequenceid=115, compaction requested=false 2024-11-20T09:01:29,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2603): Flush status journal for e765ef86c58446bc8e368358023e83a1: 2024-11-20T09:01:29,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1. 2024-11-20T09:01:29,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=118 2024-11-20T09:01:29,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.HMaster(4169): Remote procedure done, pid=118 2024-11-20T09:01:29,529 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=118, resume processing ppid=117 2024-11-20T09:01:29,529 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=118, ppid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3770 sec 2024-11-20T09:01:29,530 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees in 1.3820 sec 2024-11-20T09:01:29,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(8855): Flush requested on e765ef86c58446bc8e368358023e83a1 2024-11-20T09:01:29,682 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T09:01:29,682 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e765ef86c58446bc8e368358023e83a1 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-20T09:01:29,688 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/A/1410d9495a084237a4dfc31c3be0b624 is 50, key is test_row_0/A:col10/1732093289340/Put/seqid=0 2024-11-20T09:01:29,691 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742315_1491 (size=14391) 2024-11-20T09:01:29,757 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:29,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52804 deadline: 1732093349739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:29,758 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:29,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52840 deadline: 1732093349747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:29,758 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:29,758 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:29,758 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:01:29,759 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:29,759 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:29,759 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:01:29,771 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:29,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52854 deadline: 1732093349757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:29,773 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:29,773 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:29,773 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:01:29,877 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:29,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52804 deadline: 1732093349867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:29,878 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:29,878 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:29,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52840 deadline: 1732093349867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:29,878 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:29,878 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:01:29,878 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:29,878 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:29,878 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:01:29,888 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:29,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52854 deadline: 1732093349877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:29,888 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:29,888 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:29,888 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:01:30,092 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=129 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/A/1410d9495a084237a4dfc31c3be0b624 2024-11-20T09:01:30,095 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:30,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52804 deadline: 1732093350087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:30,096 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:30,096 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:30,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52840 deadline: 1732093350087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:30,096 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:30,096 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:01:30,097 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:30,097 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:30,097 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:01:30,101 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:30,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52854 deadline: 1732093350098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:30,102 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:30,102 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:30,102 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:01:30,108 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/B/5276e1e67c764a38a60d55e07396cdb4 is 50, key is test_row_0/B:col10/1732093289340/Put/seqid=0 2024-11-20T09:01:30,117 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742316_1492 (size=12051) 2024-11-20T09:01:30,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-20T09:01:30,297 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T09:01:30,297 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T09:01:30,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] procedure2.ProcedureExecutor(1139): Stored pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees 2024-11-20T09:01:30,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-20T09:01:30,299 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T09:01:30,300 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T09:01:30,300 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=120, ppid=119, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T09:01:30,371 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:30,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52846 deadline: 1732093350367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:30,372 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to be1eb2620e0e,34223,1732093159960 for 'test_row_0' in e765ef86c58446bc8e368358023e83a1 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2187 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-20T09:01:30,372 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:30,373 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:30,373 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:01:30,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-20T09:01:30,417 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:30,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52804 deadline: 1732093350406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:30,417 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:30,417 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:30,417 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:30,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52840 deadline: 1732093350407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:30,417 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:01:30,417 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:30,417 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:30,417 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:30,417 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:01:30,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52854 deadline: 1732093350407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:30,418 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:30,418 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:30,418 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:01:30,452 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-20T09:01:30,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1. 2024-11-20T09:01:30,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1. as already flushing 2024-11-20T09:01:30,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1. 2024-11-20T09:01:30,452 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => e765ef86c58446bc8e368358023e83a1, NAME => 'TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:30,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => e765ef86c58446bc8e368358023e83a1, NAME => 'TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:30,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e765ef86c58446bc8e368358023e83a1, NAME => 'TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e765ef86c58446bc8e368358023e83a1, NAME => 'TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:30,518 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=129 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/B/5276e1e67c764a38a60d55e07396cdb4 2024-11-20T09:01:30,525 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/C/ae362a02de4546d19d7c7b5d0bfa42a4 is 50, key is test_row_0/C:col10/1732093289340/Put/seqid=0 2024-11-20T09:01:30,526 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:30,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52884 deadline: 1732093350517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:30,526 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to be1eb2620e0e,34223,1732093159960 for 'test_row_1' in e765ef86c58446bc8e368358023e83a1 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4214 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-20T09:01:30,527 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:30,527 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:30,527 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:01:30,528 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742317_1493 (size=12051) 2024-11-20T09:01:30,529 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=129 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/C/ae362a02de4546d19d7c7b5d0bfa42a4 2024-11-20T09:01:30,532 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/A/1410d9495a084237a4dfc31c3be0b624 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/A/1410d9495a084237a4dfc31c3be0b624 2024-11-20T09:01:30,535 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/A/1410d9495a084237a4dfc31c3be0b624, entries=200, sequenceid=129, filesize=14.1 K 2024-11-20T09:01:30,536 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/B/5276e1e67c764a38a60d55e07396cdb4 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/B/5276e1e67c764a38a60d55e07396cdb4 2024-11-20T09:01:30,538 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/B/5276e1e67c764a38a60d55e07396cdb4, entries=150, sequenceid=129, filesize=11.8 K 2024-11-20T09:01:30,539 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/C/ae362a02de4546d19d7c7b5d0bfa42a4 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/C/ae362a02de4546d19d7c7b5d0bfa42a4 2024-11-20T09:01:30,544 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/C/ae362a02de4546d19d7c7b5d0bfa42a4, entries=150, sequenceid=129, filesize=11.8 K 2024-11-20T09:01:30,545 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for e765ef86c58446bc8e368358023e83a1 in 863ms, sequenceid=129, compaction requested=true 2024-11-20T09:01:30,545 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e765ef86c58446bc8e368358023e83a1: 2024-11-20T09:01:30,545 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e765ef86c58446bc8e368358023e83a1:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T09:01:30,545 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T09:01:30,545 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e765ef86c58446bc8e368358023e83a1:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T09:01:30,545 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T09:01:30,545 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T09:01:30,545 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e765ef86c58446bc8e368358023e83a1:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T09:01:30,545 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T09:01:30,545 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T09:01:30,546 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36259 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T09:01:30,546 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1541): e765ef86c58446bc8e368358023e83a1/B is initiating minor compaction (all files) 2024-11-20T09:01:30,546 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e765ef86c58446bc8e368358023e83a1/B in TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1. 2024-11-20T09:01:30,546 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/B/dcb8ea1cb8f34f3892bde30386b04ef2, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/B/9d9902e4ddea40a4888ab266b0103afb, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/B/5276e1e67c764a38a60d55e07396cdb4] into tmpdir=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp, totalSize=35.4 K 2024-11-20T09:01:30,546 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38599 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T09:01:30,547 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HStore(1541): e765ef86c58446bc8e368358023e83a1/A is initiating minor compaction (all files) 2024-11-20T09:01:30,547 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting dcb8ea1cb8f34f3892bde30386b04ef2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1732093286962 2024-11-20T09:01:30,547 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e765ef86c58446bc8e368358023e83a1/A in TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1. 2024-11-20T09:01:30,547 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/A/56fb634a8cd646e0987e34ed4d457fce, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/A/7930cf28cb00469591a56b01b26e5f3a, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/A/1410d9495a084237a4dfc31c3be0b624] into tmpdir=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp, totalSize=37.7 K 2024-11-20T09:01:30,547 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting 9d9902e4ddea40a4888ab266b0103afb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1732093288165 2024-11-20T09:01:30,548 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.Compactor(225): Compacting 56fb634a8cd646e0987e34ed4d457fce, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1732093286962 2024-11-20T09:01:30,548 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting 5276e1e67c764a38a60d55e07396cdb4, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1732093289340 2024-11-20T09:01:30,548 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7930cf28cb00469591a56b01b26e5f3a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1732093288165 2024-11-20T09:01:30,549 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1410d9495a084237a4dfc31c3be0b624, keycount=200, bloomtype=ROW, size=14.1 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1732093289340 2024-11-20T09:01:30,562 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e765ef86c58446bc8e368358023e83a1#B#compaction#416 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-20T09:01:30,563 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/B/783e6f881d73425397869ab97184c4d3 is 50, key is test_row_0/B:col10/1732093289340/Put/seqid=0 2024-11-20T09:01:30,563 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e765ef86c58446bc8e368358023e83a1#A#compaction#417 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T09:01:30,564 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/A/6568c6826c394fc98f3818a5a8d672ea is 50, key is test_row_0/A:col10/1732093289340/Put/seqid=0 2024-11-20T09:01:30,580 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742318_1494 (size=12359) 2024-11-20T09:01:30,585 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/B/783e6f881d73425397869ab97184c4d3 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/B/783e6f881d73425397869ab97184c4d3 2024-11-20T09:01:30,586 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742319_1495 (size=12359) 2024-11-20T09:01:30,589 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e765ef86c58446bc8e368358023e83a1/B of e765ef86c58446bc8e368358023e83a1 into 783e6f881d73425397869ab97184c4d3(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T09:01:30,589 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e765ef86c58446bc8e368358023e83a1: 2024-11-20T09:01:30,589 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., storeName=e765ef86c58446bc8e368358023e83a1/B, priority=13, startTime=1732093290545; duration=0sec 2024-11-20T09:01:30,589 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T09:01:30,589 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e765ef86c58446bc8e368358023e83a1:B 2024-11-20T09:01:30,590 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T09:01:30,590 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36259 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T09:01:30,590 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1541): e765ef86c58446bc8e368358023e83a1/C is initiating minor compaction (all files) 2024-11-20T09:01:30,590 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e765ef86c58446bc8e368358023e83a1/C in TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1. 2024-11-20T09:01:30,591 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/C/b606d8d0977e41758458c67f49385797, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/C/f9860385e33d4af6b0840ef0c0650bc8, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/C/ae362a02de4546d19d7c7b5d0bfa42a4] into tmpdir=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp, totalSize=35.4 K 2024-11-20T09:01:30,591 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting b606d8d0977e41758458c67f49385797, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1732093286962 2024-11-20T09:01:30,591 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting f9860385e33d4af6b0840ef0c0650bc8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1732093288165 2024-11-20T09:01:30,591 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting ae362a02de4546d19d7c7b5d0bfa42a4, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1732093289340 2024-11-20T09:01:30,596 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e765ef86c58446bc8e368358023e83a1#C#compaction#418 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T09:01:30,597 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/C/6ce0f266b563427888ede01e8c78ffe4 is 50, key is test_row_0/C:col10/1732093289340/Put/seqid=0 2024-11-20T09:01:30,600 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742320_1496 (size=12359) 2024-11-20T09:01:30,605 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-20T09:01:30,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1. 2024-11-20T09:01:30,605 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2902): Flushing e765ef86c58446bc8e368358023e83a1 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-20T09:01:30,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/A/7b7c5943defa49b29deff103c5ad984c is 50, key is test_row_0/A:col10/1732093289733/Put/seqid=0 2024-11-20T09:01:30,613 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742321_1497 (size=12151) 2024-11-20T09:01:30,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-20T09:01:30,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-20T09:01:30,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(8855): Flush requested on e765ef86c58446bc8e368358023e83a1 2024-11-20T09:01:30,930 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1. as already flushing 2024-11-20T09:01:30,991 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/A/6568c6826c394fc98f3818a5a8d672ea as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/A/6568c6826c394fc98f3818a5a8d672ea 2024-11-20T09:01:30,995 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e765ef86c58446bc8e368358023e83a1/A of e765ef86c58446bc8e368358023e83a1 into 6568c6826c394fc98f3818a5a8d672ea(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T09:01:30,995 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e765ef86c58446bc8e368358023e83a1: 2024-11-20T09:01:30,995 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., storeName=e765ef86c58446bc8e368358023e83a1/A, priority=13, startTime=1732093290545; duration=0sec 2024-11-20T09:01:30,995 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T09:01:30,995 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e765ef86c58446bc8e368358023e83a1:A 2024-11-20T09:01:30,999 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:30,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52804 deadline: 1732093350986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:30,999 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:30,999 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:30,999 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:31,000 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:01:31,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52854 deadline: 1732093350987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:31,000 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:31,000 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:31,000 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:31,000 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:01:31,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52840 deadline: 1732093350987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:31,000 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:31,000 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:31,000 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:01:31,004 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/C/6ce0f266b563427888ede01e8c78ffe4 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/C/6ce0f266b563427888ede01e8c78ffe4 2024-11-20T09:01:31,007 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e765ef86c58446bc8e368358023e83a1/C of e765ef86c58446bc8e368358023e83a1 into 6ce0f266b563427888ede01e8c78ffe4(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T09:01:31,007 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e765ef86c58446bc8e368358023e83a1: 2024-11-20T09:01:31,007 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., storeName=e765ef86c58446bc8e368358023e83a1/C, priority=13, startTime=1732093290545; duration=0sec 2024-11-20T09:01:31,007 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T09:01:31,007 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e765ef86c58446bc8e368358023e83a1:C 2024-11-20T09:01:31,014 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=153 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/A/7b7c5943defa49b29deff103c5ad984c 2024-11-20T09:01:31,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/B/668bfdebcb7b46b1bb24009a1309b41f is 50, key is test_row_0/B:col10/1732093289733/Put/seqid=0 2024-11-20T09:01:31,029 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742322_1498 (size=12151) 2024-11-20T09:01:31,029 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=153 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/B/668bfdebcb7b46b1bb24009a1309b41f 2024-11-20T09:01:31,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/C/b1934a272e314560b9466012b62afde3 is 50, key is test_row_0/C:col10/1732093289733/Put/seqid=0 2024-11-20T09:01:31,050 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742323_1499 (size=12151) 2024-11-20T09:01:31,051 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=153 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/C/b1934a272e314560b9466012b62afde3 2024-11-20T09:01:31,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/A/7b7c5943defa49b29deff103c5ad984c as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/A/7b7c5943defa49b29deff103c5ad984c 2024-11-20T09:01:31,065 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/A/7b7c5943defa49b29deff103c5ad984c, entries=150, sequenceid=153, filesize=11.9 K 2024-11-20T09:01:31,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/B/668bfdebcb7b46b1bb24009a1309b41f as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/B/668bfdebcb7b46b1bb24009a1309b41f 2024-11-20T09:01:31,074 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/B/668bfdebcb7b46b1bb24009a1309b41f, entries=150, sequenceid=153, filesize=11.9 K 2024-11-20T09:01:31,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/C/b1934a272e314560b9466012b62afde3 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/C/b1934a272e314560b9466012b62afde3 2024-11-20T09:01:31,078 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/C/b1934a272e314560b9466012b62afde3, entries=150, sequenceid=153, filesize=11.9 K 2024-11-20T09:01:31,079 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=80.51 KB/82440 for e765ef86c58446bc8e368358023e83a1 in 474ms, sequenceid=153, compaction requested=false 2024-11-20T09:01:31,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2603): Flush status journal for e765ef86c58446bc8e368358023e83a1: 2024-11-20T09:01:31,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1. 2024-11-20T09:01:31,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=120 2024-11-20T09:01:31,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.HMaster(4169): Remote procedure done, pid=120 2024-11-20T09:01:31,085 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=120, resume processing ppid=119 2024-11-20T09:01:31,085 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=120, ppid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 781 msec 2024-11-20T09:01:31,086 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees in 788 msec 2024-11-20T09:01:31,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(8855): Flush requested on e765ef86c58446bc8e368358023e83a1 2024-11-20T09:01:31,112 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T09:01:31,112 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e765ef86c58446bc8e368358023e83a1 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-20T09:01:31,122 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/A/6cd49cde118347c39d392cea52f2643f is 50, key is test_row_0/A:col10/1732093290967/Put/seqid=0 2024-11-20T09:01:31,129 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742324_1500 (size=12151) 2024-11-20T09:01:31,168 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:31,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52804 deadline: 1732093351159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:31,169 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:31,169 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:31,169 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:01:31,169 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:31,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52840 deadline: 1732093351161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:31,170 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:31,170 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:31,170 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:01:31,170 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:31,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52854 deadline: 1732093351168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:31,171 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:31,171 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:31,171 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:01:31,285 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:31,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52804 deadline: 1732093351277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:31,285 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:31,285 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:31,285 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:31,285 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:01:31,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52854 deadline: 1732093351277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:31,285 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:31,285 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:31,285 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:01:31,285 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:31,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52840 deadline: 1732093351277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:31,286 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:31,286 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:31,286 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:01:31,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-20T09:01:31,438 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T09:01:31,438 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T09:01:31,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] procedure2.ProcedureExecutor(1139): Stored pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees 2024-11-20T09:01:31,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-20T09:01:31,440 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T09:01:31,441 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T09:01:31,441 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=122, ppid=121, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T09:01:31,506 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:31,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52804 deadline: 1732093351497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:31,506 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:31,506 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:31,506 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:31,506 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:01:31,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52854 deadline: 1732093351497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:31,507 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:31,507 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:31,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52840 deadline: 1732093351497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:31,507 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:31,507 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:01:31,507 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:31,507 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:31,507 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:01:31,530 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=171 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/A/6cd49cde118347c39d392cea52f2643f 2024-11-20T09:01:31,538 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/B/e117db64b6c14c708fc2acf2b513a3d9 is 50, key is test_row_0/B:col10/1732093290967/Put/seqid=0 2024-11-20T09:01:31,542 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742325_1501 (size=12151) 2024-11-20T09:01:31,543 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=171 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/B/e117db64b6c14c708fc2acf2b513a3d9 2024-11-20T09:01:31,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-20T09:01:31,551 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/C/5087a54e2d79408c9fba20f7207cc7e5 is 50, key is test_row_0/C:col10/1732093290967/Put/seqid=0 2024-11-20T09:01:31,556 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742326_1502 (size=12151) 2024-11-20T09:01:31,594 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-20T09:01:31,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1. 2024-11-20T09:01:31,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1. as already flushing 2024-11-20T09:01:31,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1. 2024-11-20T09:01:31,595 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => e765ef86c58446bc8e368358023e83a1, NAME => 'TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:31,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => e765ef86c58446bc8e368358023e83a1, NAME => 'TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:31,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e765ef86c58446bc8e368358023e83a1, NAME => 'TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e765ef86c58446bc8e368358023e83a1, NAME => 'TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:31,747 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-20T09:01:31,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1. 2024-11-20T09:01:31,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1. as already flushing 2024-11-20T09:01:31,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1. 2024-11-20T09:01:31,748 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => e765ef86c58446bc8e368358023e83a1, NAME => 'TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:31,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => e765ef86c58446bc8e368358023e83a1, NAME => 'TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:31,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e765ef86c58446bc8e368358023e83a1, NAME => 'TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e765ef86c58446bc8e368358023e83a1, NAME => 'TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:31,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-20T09:01:31,825 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:31,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52804 deadline: 1732093351817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:31,826 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:31,826 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:31,826 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:31,826 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:01:31,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52854 deadline: 1732093351817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:31,826 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:31,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52840 deadline: 1732093351817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:31,826 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:31,826 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:31,826 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:01:31,826 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:31,826 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:31,826 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:01:31,901 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-20T09:01:31,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1. 2024-11-20T09:01:31,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1. as already flushing 2024-11-20T09:01:31,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1. 2024-11-20T09:01:31,901 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => e765ef86c58446bc8e368358023e83a1, NAME => 'TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:31,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => e765ef86c58446bc8e368358023e83a1, NAME => 'TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:31,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e765ef86c58446bc8e368358023e83a1, NAME => 'TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e765ef86c58446bc8e368358023e83a1, NAME => 'TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:31,956 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=171 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/C/5087a54e2d79408c9fba20f7207cc7e5 2024-11-20T09:01:31,960 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/A/6cd49cde118347c39d392cea52f2643f as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/A/6cd49cde118347c39d392cea52f2643f 2024-11-20T09:01:31,964 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/A/6cd49cde118347c39d392cea52f2643f, entries=150, sequenceid=171, filesize=11.9 K 2024-11-20T09:01:31,964 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/B/e117db64b6c14c708fc2acf2b513a3d9 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/B/e117db64b6c14c708fc2acf2b513a3d9 2024-11-20T09:01:31,967 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/B/e117db64b6c14c708fc2acf2b513a3d9, entries=150, sequenceid=171, filesize=11.9 K 2024-11-20T09:01:31,968 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/C/5087a54e2d79408c9fba20f7207cc7e5 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/C/5087a54e2d79408c9fba20f7207cc7e5 2024-11-20T09:01:31,971 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/C/5087a54e2d79408c9fba20f7207cc7e5, entries=150, sequenceid=171, filesize=11.9 K 2024-11-20T09:01:31,972 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for e765ef86c58446bc8e368358023e83a1 in 860ms, sequenceid=171, compaction requested=true 2024-11-20T09:01:31,972 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e765ef86c58446bc8e368358023e83a1: 2024-11-20T09:01:31,972 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e765ef86c58446bc8e368358023e83a1:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T09:01:31,972 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T09:01:31,972 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T09:01:31,972 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e765ef86c58446bc8e368358023e83a1:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T09:01:31,972 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T09:01:31,972 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e765ef86c58446bc8e368358023e83a1:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T09:01:31,972 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T09:01:31,972 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T09:01:31,973 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36661 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T09:01:31,973 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HStore(1541): e765ef86c58446bc8e368358023e83a1/A is initiating minor compaction (all files) 2024-11-20T09:01:31,973 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36661 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T09:01:31,973 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e765ef86c58446bc8e368358023e83a1/A in TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1. 2024-11-20T09:01:31,973 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1541): e765ef86c58446bc8e368358023e83a1/B is initiating minor compaction (all files) 2024-11-20T09:01:31,973 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/A/6568c6826c394fc98f3818a5a8d672ea, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/A/7b7c5943defa49b29deff103c5ad984c, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/A/6cd49cde118347c39d392cea52f2643f] into tmpdir=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp, totalSize=35.8 K 2024-11-20T09:01:31,973 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e765ef86c58446bc8e368358023e83a1/B in TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1. 2024-11-20T09:01:31,973 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/B/783e6f881d73425397869ab97184c4d3, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/B/668bfdebcb7b46b1bb24009a1309b41f, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/B/e117db64b6c14c708fc2acf2b513a3d9] into tmpdir=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp, totalSize=35.8 K 2024-11-20T09:01:31,973 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6568c6826c394fc98f3818a5a8d672ea, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1732093289340 2024-11-20T09:01:31,973 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting 783e6f881d73425397869ab97184c4d3, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1732093289340 2024-11-20T09:01:31,974 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7b7c5943defa49b29deff103c5ad984c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1732093289733 2024-11-20T09:01:31,974 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting 668bfdebcb7b46b1bb24009a1309b41f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1732093289733 2024-11-20T09:01:31,974 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6cd49cde118347c39d392cea52f2643f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1732093290956 2024-11-20T09:01:31,974 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting e117db64b6c14c708fc2acf2b513a3d9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1732093290956 2024-11-20T09:01:31,979 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e765ef86c58446bc8e368358023e83a1#A#compaction#425 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-20T09:01:31,980 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/A/40ecb445f26e41ec898344699923b391 is 50, key is test_row_0/A:col10/1732093290967/Put/seqid=0 2024-11-20T09:01:31,980 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e765ef86c58446bc8e368358023e83a1#B#compaction#426 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T09:01:31,981 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/B/08e93114d7d34d6abfe76efda84d5177 is 50, key is test_row_0/B:col10/1732093290967/Put/seqid=0 2024-11-20T09:01:31,983 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742327_1503 (size=12561) 2024-11-20T09:01:31,985 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742328_1504 (size=12561) 2024-11-20T09:01:31,988 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/B/08e93114d7d34d6abfe76efda84d5177 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/B/08e93114d7d34d6abfe76efda84d5177 2024-11-20T09:01:31,992 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e765ef86c58446bc8e368358023e83a1/B of e765ef86c58446bc8e368358023e83a1 into 08e93114d7d34d6abfe76efda84d5177(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T09:01:31,992 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e765ef86c58446bc8e368358023e83a1: 2024-11-20T09:01:31,992 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., storeName=e765ef86c58446bc8e368358023e83a1/B, priority=13, startTime=1732093291972; duration=0sec 2024-11-20T09:01:31,992 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T09:01:31,992 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e765ef86c58446bc8e368358023e83a1:B 2024-11-20T09:01:31,992 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T09:01:31,993 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36661 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T09:01:31,993 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1541): e765ef86c58446bc8e368358023e83a1/C is initiating minor compaction (all files) 2024-11-20T09:01:31,993 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e765ef86c58446bc8e368358023e83a1/C in TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1. 2024-11-20T09:01:31,993 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/C/6ce0f266b563427888ede01e8c78ffe4, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/C/b1934a272e314560b9466012b62afde3, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/C/5087a54e2d79408c9fba20f7207cc7e5] into tmpdir=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp, totalSize=35.8 K 2024-11-20T09:01:31,993 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting 6ce0f266b563427888ede01e8c78ffe4, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1732093289340 2024-11-20T09:01:31,993 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting b1934a272e314560b9466012b62afde3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1732093289733 2024-11-20T09:01:31,994 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting 5087a54e2d79408c9fba20f7207cc7e5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1732093290956 2024-11-20T09:01:32,001 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e765ef86c58446bc8e368358023e83a1#C#compaction#427 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T09:01:32,002 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/C/eb23e2df7043431eb796f695d2912a64 is 50, key is test_row_0/C:col10/1732093290967/Put/seqid=0 2024-11-20T09:01:32,006 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742329_1505 (size=12561) 2024-11-20T09:01:32,054 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-20T09:01:32,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1. 2024-11-20T09:01:32,054 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2902): Flushing e765ef86c58446bc8e368358023e83a1 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-20T09:01:32,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/A/ee8140cc1c4849c6824b05bdf362781b is 50, key is test_row_0/A:col10/1732093291160/Put/seqid=0 2024-11-20T09:01:32,062 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742330_1506 (size=12151) 2024-11-20T09:01:32,062 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=192 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/A/ee8140cc1c4849c6824b05bdf362781b 2024-11-20T09:01:32,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-20T09:01:32,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/B/bcc8a034088348aba915b67edd0f4567 is 50, key is test_row_0/B:col10/1732093291160/Put/seqid=0 2024-11-20T09:01:32,073 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742331_1507 (size=12151) 2024-11-20T09:01:32,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(8855): Flush requested on e765ef86c58446bc8e368358023e83a1 2024-11-20T09:01:32,341 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1. as already flushing 2024-11-20T09:01:32,388 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/A/40ecb445f26e41ec898344699923b391 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/A/40ecb445f26e41ec898344699923b391 2024-11-20T09:01:32,391 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e765ef86c58446bc8e368358023e83a1/A of e765ef86c58446bc8e368358023e83a1 into 40ecb445f26e41ec898344699923b391(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T09:01:32,391 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e765ef86c58446bc8e368358023e83a1: 2024-11-20T09:01:32,392 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., storeName=e765ef86c58446bc8e368358023e83a1/A, priority=13, startTime=1732093291972; duration=0sec 2024-11-20T09:01:32,392 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T09:01:32,392 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e765ef86c58446bc8e368358023e83a1:A 2024-11-20T09:01:32,395 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:32,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52854 deadline: 1732093352382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:32,395 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:32,395 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:32,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52840 deadline: 1732093352383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:32,395 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:32,395 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:01:32,395 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:32,395 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:32,395 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:01:32,396 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:32,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52804 deadline: 1732093352384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:32,396 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:32,396 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:32,396 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:01:32,406 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:32,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52846 deadline: 1732093352397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:32,406 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to be1eb2620e0e,34223,1732093159960 for 'test_row_0' in e765ef86c58446bc8e368358023e83a1 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4222 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-20T09:01:32,407 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:32,407 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:32,407 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:01:32,410 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/C/eb23e2df7043431eb796f695d2912a64 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/C/eb23e2df7043431eb796f695d2912a64 2024-11-20T09:01:32,413 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e765ef86c58446bc8e368358023e83a1/C of e765ef86c58446bc8e368358023e83a1 into eb23e2df7043431eb796f695d2912a64(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T09:01:32,413 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e765ef86c58446bc8e368358023e83a1: 2024-11-20T09:01:32,413 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., storeName=e765ef86c58446bc8e368358023e83a1/C, priority=13, startTime=1732093291972; duration=0sec 2024-11-20T09:01:32,413 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T09:01:32,413 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e765ef86c58446bc8e368358023e83a1:C 2024-11-20T09:01:32,473 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=192 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/B/bcc8a034088348aba915b67edd0f4567 2024-11-20T09:01:32,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/C/8c6e4efb5b5a44b79deb05f1dd364f6a is 50, key is test_row_0/C:col10/1732093291160/Put/seqid=0 2024-11-20T09:01:32,484 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742332_1508 (size=12151) 2024-11-20T09:01:32,501 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:32,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52854 deadline: 1732093352497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:32,502 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:32,502 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:32,502 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:01:32,513 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:32,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52840 deadline: 1732093352506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:32,513 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:32,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52804 deadline: 1732093352506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:32,513 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:32,513 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:32,514 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:01:32,514 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:32,514 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:32,514 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:01:32,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-20T09:01:32,711 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:32,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52854 deadline: 1732093352707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:32,712 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:32,712 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:32,712 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:01:32,723 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:32,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52840 deadline: 1732093352716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:32,723 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:32,723 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:32,723 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:32,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52804 deadline: 1732093352717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:32,723 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:01:32,724 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:32,724 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:32,724 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:01:32,884 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=192 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/C/8c6e4efb5b5a44b79deb05f1dd364f6a 2024-11-20T09:01:32,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/A/ee8140cc1c4849c6824b05bdf362781b as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/A/ee8140cc1c4849c6824b05bdf362781b 2024-11-20T09:01:32,891 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/A/ee8140cc1c4849c6824b05bdf362781b, entries=150, sequenceid=192, filesize=11.9 K 2024-11-20T09:01:32,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/B/bcc8a034088348aba915b67edd0f4567 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/B/bcc8a034088348aba915b67edd0f4567 2024-11-20T09:01:32,895 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/B/bcc8a034088348aba915b67edd0f4567, entries=150, sequenceid=192, filesize=11.9 K 2024-11-20T09:01:32,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/C/8c6e4efb5b5a44b79deb05f1dd364f6a as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/C/8c6e4efb5b5a44b79deb05f1dd364f6a 2024-11-20T09:01:32,898 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/C/8c6e4efb5b5a44b79deb05f1dd364f6a, entries=150, sequenceid=192, filesize=11.9 K 2024-11-20T09:01:32,899 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for e765ef86c58446bc8e368358023e83a1 in 845ms, sequenceid=192, compaction requested=false 2024-11-20T09:01:32,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2603): Flush status journal for e765ef86c58446bc8e368358023e83a1: 2024-11-20T09:01:32,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1. 2024-11-20T09:01:32,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=122 2024-11-20T09:01:32,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.HMaster(4169): Remote procedure done, pid=122 2024-11-20T09:01:32,902 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=122, resume processing ppid=121 2024-11-20T09:01:32,902 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=122, ppid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4600 sec 2024-11-20T09:01:32,904 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees in 1.4650 sec 2024-11-20T09:01:33,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(8855): Flush requested on e765ef86c58446bc8e368358023e83a1 2024-11-20T09:01:33,021 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T09:01:33,021 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e765ef86c58446bc8e368358023e83a1 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-20T09:01:33,026 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/A/7d28d4f828424e8ab750783d8808a5b9 is 50, key is test_row_0/A:col10/1732093293020/Put/seqid=0 2024-11-20T09:01:33,030 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742333_1509 (size=14541) 2024-11-20T09:01:33,088 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:33,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52840 deadline: 1732093353082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:33,089 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:33,089 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:33,089 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:01:33,094 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:33,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52804 deadline: 1732093353084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:33,094 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:33,094 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:33,094 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:01:33,098 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:33,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52854 deadline: 1732093353085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:33,098 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:33,098 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:33,098 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:01:33,206 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:33,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52840 deadline: 1732093353196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:33,206 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:33,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52804 deadline: 1732093353197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:33,206 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:33,206 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:33,206 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:01:33,207 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:33,207 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:33,207 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:01:33,211 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:33,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52854 deadline: 1732093353207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:33,211 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:33,211 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:33,211 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:01:33,428 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:33,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52840 deadline: 1732093353417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:33,428 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:33,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52804 deadline: 1732093353417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:33,428 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:33,428 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:33,428 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:01:33,429 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:33,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52854 deadline: 1732093353417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:33,429 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:33,429 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:33,429 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:01:33,429 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:33,429 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:33,429 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:01:33,431 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=211 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/A/7d28d4f828424e8ab750783d8808a5b9 2024-11-20T09:01:33,437 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/B/d3a78d5597704ad981fad51330cc19e3 is 50, key is test_row_0/B:col10/1732093293020/Put/seqid=0 2024-11-20T09:01:33,441 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742334_1510 (size=12151) 2024-11-20T09:01:33,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-20T09:01:33,587 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T09:01:33,588 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T09:01:33,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] procedure2.ProcedureExecutor(1139): Stored pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees 2024-11-20T09:01:33,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-20T09:01:33,590 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T09:01:33,591 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T09:01:33,591 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=124, ppid=123, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T09:01:33,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-20T09:01:33,743 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-20T09:01:33,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1. 2024-11-20T09:01:33,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1. as already flushing 2024-11-20T09:01:33,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1. 2024-11-20T09:01:33,743 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => e765ef86c58446bc8e368358023e83a1, NAME => 'TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:33,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => e765ef86c58446bc8e368358023e83a1, NAME => 'TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:33,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e765ef86c58446bc8e368358023e83a1, NAME => 'TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e765ef86c58446bc8e368358023e83a1, NAME => 'TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:33,747 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:33,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52840 deadline: 1732093353736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:33,747 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:33,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52804 deadline: 1732093353736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:33,748 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:33,748 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:33,748 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:01:33,748 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:33,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52854 deadline: 1732093353736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:33,748 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:33,748 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:33,748 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:01:33,749 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:33,749 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:33,749 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:01:33,842 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=211 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/B/d3a78d5597704ad981fad51330cc19e3 2024-11-20T09:01:33,848 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/C/afbb338dbfbc40f1b3be7078dcc0d9ee is 50, key is test_row_0/C:col10/1732093293020/Put/seqid=0 2024-11-20T09:01:33,852 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742335_1511 (size=12151) 2024-11-20T09:01:33,896 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-20T09:01:33,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1. 2024-11-20T09:01:33,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1. as already flushing 2024-11-20T09:01:33,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1. 2024-11-20T09:01:33,896 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => e765ef86c58446bc8e368358023e83a1, NAME => 'TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:33,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => e765ef86c58446bc8e368358023e83a1, NAME => 'TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:33,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e765ef86c58446bc8e368358023e83a1, NAME => 'TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e765ef86c58446bc8e368358023e83a1, NAME => 'TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:33,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-20T09:01:34,049 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-20T09:01:34,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1. 2024-11-20T09:01:34,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1. as already flushing 2024-11-20T09:01:34,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1. 2024-11-20T09:01:34,049 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => e765ef86c58446bc8e368358023e83a1, NAME => 'TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:34,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => e765ef86c58446bc8e368358023e83a1, NAME => 'TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:34,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e765ef86c58446bc8e368358023e83a1, NAME => 'TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e765ef86c58446bc8e368358023e83a1, NAME => 'TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:34,202 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-20T09:01:34,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1. 2024-11-20T09:01:34,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1. as already flushing 2024-11-20T09:01:34,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1. 2024-11-20T09:01:34,202 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => e765ef86c58446bc8e368358023e83a1, NAME => 'TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:34,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => e765ef86c58446bc8e368358023e83a1, NAME => 'TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:34,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e765ef86c58446bc8e368358023e83a1, NAME => 'TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e765ef86c58446bc8e368358023e83a1, NAME => 'TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:34,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-20T09:01:34,253 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=211 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/C/afbb338dbfbc40f1b3be7078dcc0d9ee 2024-11-20T09:01:34,257 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/A/7d28d4f828424e8ab750783d8808a5b9 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/A/7d28d4f828424e8ab750783d8808a5b9 2024-11-20T09:01:34,260 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/A/7d28d4f828424e8ab750783d8808a5b9, entries=200, sequenceid=211, filesize=14.2 K 2024-11-20T09:01:34,261 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/B/d3a78d5597704ad981fad51330cc19e3 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/B/d3a78d5597704ad981fad51330cc19e3 2024-11-20T09:01:34,264 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/B/d3a78d5597704ad981fad51330cc19e3, entries=150, sequenceid=211, filesize=11.9 K 2024-11-20T09:01:34,265 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/C/afbb338dbfbc40f1b3be7078dcc0d9ee as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/C/afbb338dbfbc40f1b3be7078dcc0d9ee 2024-11-20T09:01:34,265 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:34,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52840 deadline: 1732093354256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:34,266 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:34,266 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:34,266 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:34,266 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:01:34,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52804 deadline: 1732093354256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:34,266 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:34,266 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:34,266 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:34,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52854 deadline: 1732093354257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:34,266 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:01:34,266 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:34,266 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:34,267 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:01:34,268 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/C/afbb338dbfbc40f1b3be7078dcc0d9ee, entries=150, sequenceid=211, filesize=11.9 K 2024-11-20T09:01:34,269 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for e765ef86c58446bc8e368358023e83a1 in 1248ms, sequenceid=211, compaction requested=true 2024-11-20T09:01:34,269 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e765ef86c58446bc8e368358023e83a1: 2024-11-20T09:01:34,269 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e765ef86c58446bc8e368358023e83a1:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T09:01:34,269 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T09:01:34,269 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e765ef86c58446bc8e368358023e83a1:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T09:01:34,269 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T09:01:34,269 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e765ef86c58446bc8e368358023e83a1:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T09:01:34,269 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T09:01:34,269 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T09:01:34,269 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T09:01:34,270 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39253 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T09:01:34,270 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T09:01:34,270 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HStore(1541): e765ef86c58446bc8e368358023e83a1/A is initiating minor compaction (all files) 2024-11-20T09:01:34,270 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1541): e765ef86c58446bc8e368358023e83a1/B is initiating minor compaction (all files) 2024-11-20T09:01:34,270 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e765ef86c58446bc8e368358023e83a1/A in TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1. 2024-11-20T09:01:34,270 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e765ef86c58446bc8e368358023e83a1/B in TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1. 2024-11-20T09:01:34,270 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/A/40ecb445f26e41ec898344699923b391, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/A/ee8140cc1c4849c6824b05bdf362781b, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/A/7d28d4f828424e8ab750783d8808a5b9] into tmpdir=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp, totalSize=38.3 K 2024-11-20T09:01:34,270 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/B/08e93114d7d34d6abfe76efda84d5177, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/B/bcc8a034088348aba915b67edd0f4567, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/B/d3a78d5597704ad981fad51330cc19e3] into tmpdir=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp, totalSize=36.0 K 2024-11-20T09:01:34,270 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.Compactor(225): Compacting 40ecb445f26e41ec898344699923b391, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1732093290956 2024-11-20T09:01:34,271 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting 08e93114d7d34d6abfe76efda84d5177, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1732093290956 2024-11-20T09:01:34,271 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.Compactor(225): Compacting ee8140cc1c4849c6824b05bdf362781b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1732093291156 2024-11-20T09:01:34,271 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting bcc8a034088348aba915b67edd0f4567, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1732093291156 2024-11-20T09:01:34,271 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7d28d4f828424e8ab750783d8808a5b9, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1732093292380 2024-11-20T09:01:34,272 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting d3a78d5597704ad981fad51330cc19e3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1732093292380 2024-11-20T09:01:34,278 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e765ef86c58446bc8e368358023e83a1#B#compaction#434 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T09:01:34,278 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/B/ea3c53c0b9d94ef18438e032036f69b6 is 50, key is test_row_0/B:col10/1732093293020/Put/seqid=0 2024-11-20T09:01:34,282 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e765ef86c58446bc8e368358023e83a1#A#compaction#435 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T09:01:34,283 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/A/3ded0b6642764516989f24b70464f481 is 50, key is test_row_0/A:col10/1732093293020/Put/seqid=0 2024-11-20T09:01:34,296 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742336_1512 (size=12663) 2024-11-20T09:01:34,303 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742337_1513 (size=12663) 2024-11-20T09:01:34,303 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/B/ea3c53c0b9d94ef18438e032036f69b6 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/B/ea3c53c0b9d94ef18438e032036f69b6 2024-11-20T09:01:34,307 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e765ef86c58446bc8e368358023e83a1/B of e765ef86c58446bc8e368358023e83a1 into ea3c53c0b9d94ef18438e032036f69b6(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T09:01:34,307 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e765ef86c58446bc8e368358023e83a1: 2024-11-20T09:01:34,307 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., storeName=e765ef86c58446bc8e368358023e83a1/B, priority=13, startTime=1732093294269; duration=0sec 2024-11-20T09:01:34,307 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T09:01:34,307 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e765ef86c58446bc8e368358023e83a1:B 2024-11-20T09:01:34,307 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T09:01:34,310 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T09:01:34,310 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1541): e765ef86c58446bc8e368358023e83a1/C is initiating minor compaction (all files) 2024-11-20T09:01:34,310 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e765ef86c58446bc8e368358023e83a1/C in TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1. 2024-11-20T09:01:34,310 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/C/eb23e2df7043431eb796f695d2912a64, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/C/8c6e4efb5b5a44b79deb05f1dd364f6a, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/C/afbb338dbfbc40f1b3be7078dcc0d9ee] into tmpdir=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp, totalSize=36.0 K 2024-11-20T09:01:34,310 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting eb23e2df7043431eb796f695d2912a64, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1732093290956 2024-11-20T09:01:34,311 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting 8c6e4efb5b5a44b79deb05f1dd364f6a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1732093291156 2024-11-20T09:01:34,311 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting afbb338dbfbc40f1b3be7078dcc0d9ee, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1732093292380 2024-11-20T09:01:34,317 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e765ef86c58446bc8e368358023e83a1#C#compaction#436 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T09:01:34,317 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/C/3e86b9bd75fa4391a1cab6fef518dd8b is 50, key is test_row_0/C:col10/1732093293020/Put/seqid=0 2024-11-20T09:01:34,329 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742338_1514 (size=12663) 2024-11-20T09:01:34,355 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-20T09:01:34,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1. 2024-11-20T09:01:34,355 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2902): Flushing e765ef86c58446bc8e368358023e83a1 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-20T09:01:34,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/A/eb633c56abaa480ca9dc42ee7c2757be is 50, key is test_row_0/A:col10/1732093293076/Put/seqid=0 2024-11-20T09:01:34,364 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742339_1515 (size=12151) 2024-11-20T09:01:34,368 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=231 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/A/eb633c56abaa480ca9dc42ee7c2757be 2024-11-20T09:01:34,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/B/1be6026e6cd7422bb1f4c335a8d4dcb5 is 50, key is test_row_0/B:col10/1732093293076/Put/seqid=0 2024-11-20T09:01:34,378 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742340_1516 (size=12151) 2024-11-20T09:01:34,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(8855): Flush requested on e765ef86c58446bc8e368358023e83a1 2024-11-20T09:01:34,562 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1. as already flushing 2024-11-20T09:01:34,697 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:34,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52884 deadline: 1732093354684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:34,698 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:34,698 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:34,698 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:01:34,707 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/A/3ded0b6642764516989f24b70464f481 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/A/3ded0b6642764516989f24b70464f481 2024-11-20T09:01:34,711 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e765ef86c58446bc8e368358023e83a1/A of e765ef86c58446bc8e368358023e83a1 into 3ded0b6642764516989f24b70464f481(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T09:01:34,711 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e765ef86c58446bc8e368358023e83a1: 2024-11-20T09:01:34,711 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., storeName=e765ef86c58446bc8e368358023e83a1/A, priority=13, startTime=1732093294269; duration=0sec 2024-11-20T09:01:34,711 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T09:01:34,711 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e765ef86c58446bc8e368358023e83a1:A 2024-11-20T09:01:34,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-20T09:01:34,733 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/C/3e86b9bd75fa4391a1cab6fef518dd8b as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/C/3e86b9bd75fa4391a1cab6fef518dd8b 2024-11-20T09:01:34,736 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e765ef86c58446bc8e368358023e83a1/C of e765ef86c58446bc8e368358023e83a1 into 3e86b9bd75fa4391a1cab6fef518dd8b(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T09:01:34,737 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e765ef86c58446bc8e368358023e83a1: 2024-11-20T09:01:34,737 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., storeName=e765ef86c58446bc8e368358023e83a1/C, priority=13, startTime=1732093294269; duration=0sec 2024-11-20T09:01:34,737 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T09:01:34,737 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e765ef86c58446bc8e368358023e83a1:C 2024-11-20T09:01:34,779 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=231 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/B/1be6026e6cd7422bb1f4c335a8d4dcb5 2024-11-20T09:01:34,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/C/174ce5669bac443c8a3a58c446e4e80c is 50, key is test_row_0/C:col10/1732093293076/Put/seqid=0 2024-11-20T09:01:34,790 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742341_1517 (size=12151) 2024-11-20T09:01:34,814 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:34,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52884 deadline: 1732093354807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:34,815 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:34,815 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:34,815 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:01:35,024 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:35,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52884 deadline: 1732093355017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:35,024 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:35,024 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:35,024 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:01:35,191 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=231 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/C/174ce5669bac443c8a3a58c446e4e80c 2024-11-20T09:01:35,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/A/eb633c56abaa480ca9dc42ee7c2757be as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/A/eb633c56abaa480ca9dc42ee7c2757be 2024-11-20T09:01:35,198 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/A/eb633c56abaa480ca9dc42ee7c2757be, entries=150, sequenceid=231, filesize=11.9 K 2024-11-20T09:01:35,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/B/1be6026e6cd7422bb1f4c335a8d4dcb5 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/B/1be6026e6cd7422bb1f4c335a8d4dcb5 2024-11-20T09:01:35,201 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/B/1be6026e6cd7422bb1f4c335a8d4dcb5, entries=150, sequenceid=231, filesize=11.9 K 2024-11-20T09:01:35,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/C/174ce5669bac443c8a3a58c446e4e80c as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/C/174ce5669bac443c8a3a58c446e4e80c 2024-11-20T09:01:35,206 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/C/174ce5669bac443c8a3a58c446e4e80c, entries=150, sequenceid=231, filesize=11.9 K 2024-11-20T09:01:35,207 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=93.93 KB/96180 for e765ef86c58446bc8e368358023e83a1 in 852ms, sequenceid=231, compaction requested=false 2024-11-20T09:01:35,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2603): Flush status journal for e765ef86c58446bc8e368358023e83a1: 2024-11-20T09:01:35,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1. 2024-11-20T09:01:35,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=124 2024-11-20T09:01:35,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.HMaster(4169): Remote procedure done, pid=124 2024-11-20T09:01:35,209 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=124, resume processing ppid=123 2024-11-20T09:01:35,209 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=124, ppid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6170 sec 2024-11-20T09:01:35,211 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees in 1.6220 sec 2024-11-20T09:01:35,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(8855): Flush requested on e765ef86c58446bc8e368358023e83a1 2024-11-20T09:01:35,280 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T09:01:35,281 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e765ef86c58446bc8e368358023e83a1 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-20T09:01:35,285 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/A/4738cbe7563a442f859e5a0216e1b04f is 50, key is test_row_0/A:col10/1732093295279/Put/seqid=0 2024-11-20T09:01:35,289 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742342_1518 (size=14541) 2024-11-20T09:01:35,339 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:35,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52840 deadline: 1732093355325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:35,340 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:35,340 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:35,340 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:35,340 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:01:35,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52854 deadline: 1732093355326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:35,340 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:35,340 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:35,340 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:01:35,341 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:35,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52804 deadline: 1732093355330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:35,341 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:35,341 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:35,341 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:01:35,344 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:35,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52884 deadline: 1732093355337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:35,345 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:35,345 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:35,345 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:01:35,456 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:35,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52840 deadline: 1732093355446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:35,456 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:35,456 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:35,457 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:35,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52854 deadline: 1732093355447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:35,457 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:01:35,457 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:35,457 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:35,457 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:35,457 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:01:35,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52804 deadline: 1732093355447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:35,457 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:35,457 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:35,457 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:01:35,675 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:35,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52840 deadline: 1732093355666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:35,675 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:35,675 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:35,675 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:35,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52854 deadline: 1732093355667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:35,675 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:01:35,675 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:35,675 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:35,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52804 deadline: 1732093355667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:35,675 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:35,675 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:01:35,676 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:35,676 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:35,676 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:01:35,689 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=251 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/A/4738cbe7563a442f859e5a0216e1b04f 2024-11-20T09:01:35,696 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/B/fce689c164c34bed937078efb069e61c is 50, key is test_row_0/B:col10/1732093295279/Put/seqid=0 2024-11-20T09:01:35,699 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742343_1519 (size=12151) 2024-11-20T09:01:35,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-20T09:01:35,737 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T09:01:35,737 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T09:01:35,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] procedure2.ProcedureExecutor(1139): Stored pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees 2024-11-20T09:01:35,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-20T09:01:35,739 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T09:01:35,739 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T09:01:35,739 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=126, ppid=125, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T09:01:35,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-20T09:01:35,864 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:35,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52884 deadline: 1732093355857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:35,865 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:35,865 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:35,865 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:01:35,892 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-20T09:01:35,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1. 2024-11-20T09:01:35,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1. as already flushing 2024-11-20T09:01:35,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1. 2024-11-20T09:01:35,892 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => e765ef86c58446bc8e368358023e83a1, NAME => 'TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:35,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => e765ef86c58446bc8e368358023e83a1, NAME => 'TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:35,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e765ef86c58446bc8e368358023e83a1, NAME => 'TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e765ef86c58446bc8e368358023e83a1, NAME => 'TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:35,998 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:35,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52840 deadline: 1732093355987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:35,998 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:35,998 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:35,999 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:35,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52854 deadline: 1732093355987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:35,999 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:01:35,999 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:35,999 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:35,999 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:35,999 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:01:35,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52804 deadline: 1732093355987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:35,999 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:35,999 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:35,999 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:01:36,045 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-20T09:01:36,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1. 2024-11-20T09:01:36,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1. as already flushing 2024-11-20T09:01:36,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1. 2024-11-20T09:01:36,045 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => e765ef86c58446bc8e368358023e83a1, NAME => 'TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:36,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => e765ef86c58446bc8e368358023e83a1, NAME => 'TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:36,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e765ef86c58446bc8e368358023e83a1, NAME => 'TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e765ef86c58446bc8e368358023e83a1, NAME => 'TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:36,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-20T09:01:36,100 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=251 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/B/fce689c164c34bed937078efb069e61c 2024-11-20T09:01:36,106 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/C/aabf66da690a4c57a09c79d93bc18a90 is 50, key is test_row_0/C:col10/1732093295279/Put/seqid=0 2024-11-20T09:01:36,109 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742344_1520 (size=12151) 2024-11-20T09:01:36,197 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-20T09:01:36,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1. 2024-11-20T09:01:36,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1. as already flushing 2024-11-20T09:01:36,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1. 2024-11-20T09:01:36,198 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => e765ef86c58446bc8e368358023e83a1, NAME => 'TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:36,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => e765ef86c58446bc8e368358023e83a1, NAME => 'TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:36,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e765ef86c58446bc8e368358023e83a1, NAME => 'TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e765ef86c58446bc8e368358023e83a1, NAME => 'TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:36,350 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-20T09:01:36,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1. 2024-11-20T09:01:36,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1. as already flushing 2024-11-20T09:01:36,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1. 2024-11-20T09:01:36,351 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => e765ef86c58446bc8e368358023e83a1, NAME => 'TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:36,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => e765ef86c58446bc8e368358023e83a1, NAME => 'TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:36,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e765ef86c58446bc8e368358023e83a1, NAME => 'TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e765ef86c58446bc8e368358023e83a1, NAME => 'TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:36,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-20T09:01:36,447 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:36,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52846 deadline: 1732093356436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:36,447 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to be1eb2620e0e,34223,1732093159960 for 'test_row_0' in e765ef86c58446bc8e368358023e83a1 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8263 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-20T09:01:36,448 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:36,448 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:36,448 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:01:36,503 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-20T09:01:36,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1. 2024-11-20T09:01:36,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1. as already flushing 2024-11-20T09:01:36,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1. 2024-11-20T09:01:36,504 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => e765ef86c58446bc8e368358023e83a1, NAME => 'TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:36,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => e765ef86c58446bc8e368358023e83a1, NAME => 'TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:36,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e765ef86c58446bc8e368358023e83a1, NAME => 'TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e765ef86c58446bc8e368358023e83a1, NAME => 'TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:36,511 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=251 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/C/aabf66da690a4c57a09c79d93bc18a90 2024-11-20T09:01:36,515 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/A/4738cbe7563a442f859e5a0216e1b04f as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/A/4738cbe7563a442f859e5a0216e1b04f 2024-11-20T09:01:36,517 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:36,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52840 deadline: 1732093356507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:36,517 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:36,518 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:36,518 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:36,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52854 deadline: 1732093356507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:36,518 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:01:36,518 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:36,518 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:36,518 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:36,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52804 deadline: 1732093356507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:36,518 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:01:36,518 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/A/4738cbe7563a442f859e5a0216e1b04f, entries=200, sequenceid=251, filesize=14.2 K 2024-11-20T09:01:36,518 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:36,518 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:36,518 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:01:36,519 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/B/fce689c164c34bed937078efb069e61c as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/B/fce689c164c34bed937078efb069e61c 2024-11-20T09:01:36,522 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/B/fce689c164c34bed937078efb069e61c, entries=150, sequenceid=251, filesize=11.9 K 2024-11-20T09:01:36,522 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/C/aabf66da690a4c57a09c79d93bc18a90 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/C/aabf66da690a4c57a09c79d93bc18a90 2024-11-20T09:01:36,525 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/C/aabf66da690a4c57a09c79d93bc18a90, entries=150, sequenceid=251, filesize=11.9 K 2024-11-20T09:01:36,526 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for e765ef86c58446bc8e368358023e83a1 in 1246ms, sequenceid=251, compaction requested=true 2024-11-20T09:01:36,526 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e765ef86c58446bc8e368358023e83a1: 2024-11-20T09:01:36,526 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e765ef86c58446bc8e368358023e83a1:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T09:01:36,526 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T09:01:36,526 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e765ef86c58446bc8e368358023e83a1:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T09:01:36,526 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T09:01:36,526 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e765ef86c58446bc8e368358023e83a1:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T09:01:36,526 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-20T09:01:36,526 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T09:01:36,526 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T09:01:36,527 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T09:01:36,527 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39355 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T09:01:36,527 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1541): e765ef86c58446bc8e368358023e83a1/B is initiating minor compaction (all files) 2024-11-20T09:01:36,527 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HStore(1541): e765ef86c58446bc8e368358023e83a1/A is initiating minor compaction (all files) 2024-11-20T09:01:36,527 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e765ef86c58446bc8e368358023e83a1/B in TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1. 2024-11-20T09:01:36,527 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e765ef86c58446bc8e368358023e83a1/A in TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1. 2024-11-20T09:01:36,527 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/B/ea3c53c0b9d94ef18438e032036f69b6, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/B/1be6026e6cd7422bb1f4c335a8d4dcb5, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/B/fce689c164c34bed937078efb069e61c] into tmpdir=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp, totalSize=36.1 K 2024-11-20T09:01:36,527 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/A/3ded0b6642764516989f24b70464f481, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/A/eb633c56abaa480ca9dc42ee7c2757be, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/A/4738cbe7563a442f859e5a0216e1b04f] into tmpdir=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp, totalSize=38.4 K 2024-11-20T09:01:36,529 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting ea3c53c0b9d94ef18438e032036f69b6, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1732093292380 2024-11-20T09:01:36,529 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3ded0b6642764516989f24b70464f481, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1732093292380 2024-11-20T09:01:36,530 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting 1be6026e6cd7422bb1f4c335a8d4dcb5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1732093293076 2024-11-20T09:01:36,530 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.Compactor(225): Compacting eb633c56abaa480ca9dc42ee7c2757be, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1732093293076 2024-11-20T09:01:36,530 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting fce689c164c34bed937078efb069e61c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1732093294636 2024-11-20T09:01:36,530 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4738cbe7563a442f859e5a0216e1b04f, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1732093294636 2024-11-20T09:01:36,542 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e765ef86c58446bc8e368358023e83a1#B#compaction#443 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-20T09:01:36,543 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e765ef86c58446bc8e368358023e83a1#A#compaction#444 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T09:01:36,543 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/B/cf9f2ce06d4842a68c7b3dcd66beb210 is 50, key is test_row_0/B:col10/1732093295279/Put/seqid=0 2024-11-20T09:01:36,543 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/A/f2f72fc1bd5945ea81c3160a0b220707 is 50, key is test_row_0/A:col10/1732093295279/Put/seqid=0 2024-11-20T09:01:36,549 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742346_1522 (size=12765) 2024-11-20T09:01:36,550 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742345_1521 (size=12765) 2024-11-20T09:01:36,656 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-20T09:01:36,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1. 2024-11-20T09:01:36,656 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2902): Flushing e765ef86c58446bc8e368358023e83a1 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-20T09:01:36,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/A/e52ac2ecf4bd4820823c83c6b97a4e3e is 50, key is test_row_0/A:col10/1732093295314/Put/seqid=0 2024-11-20T09:01:36,665 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742347_1523 (size=12301) 2024-11-20T09:01:36,871 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1. as already flushing 2024-11-20T09:01:36,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(8855): Flush requested on e765ef86c58446bc8e368358023e83a1 2024-11-20T09:01:36,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-20T09:01:36,954 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/A/f2f72fc1bd5945ea81c3160a0b220707 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/A/f2f72fc1bd5945ea81c3160a0b220707 2024-11-20T09:01:36,954 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/B/cf9f2ce06d4842a68c7b3dcd66beb210 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/B/cf9f2ce06d4842a68c7b3dcd66beb210 2024-11-20T09:01:36,958 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e765ef86c58446bc8e368358023e83a1/A of e765ef86c58446bc8e368358023e83a1 into f2f72fc1bd5945ea81c3160a0b220707(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T09:01:36,958 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e765ef86c58446bc8e368358023e83a1: 2024-11-20T09:01:36,958 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., storeName=e765ef86c58446bc8e368358023e83a1/A, priority=13, startTime=1732093296526; duration=0sec 2024-11-20T09:01:36,958 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T09:01:36,958 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e765ef86c58446bc8e368358023e83a1:A 2024-11-20T09:01:36,958 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T09:01:36,959 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T09:01:36,959 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HStore(1541): e765ef86c58446bc8e368358023e83a1/C is initiating minor compaction (all files) 2024-11-20T09:01:36,959 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e765ef86c58446bc8e368358023e83a1/C in TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1. 2024-11-20T09:01:36,959 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/C/3e86b9bd75fa4391a1cab6fef518dd8b, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/C/174ce5669bac443c8a3a58c446e4e80c, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/C/aabf66da690a4c57a09c79d93bc18a90] into tmpdir=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp, totalSize=36.1 K 2024-11-20T09:01:36,959 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3e86b9bd75fa4391a1cab6fef518dd8b, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1732093292380 2024-11-20T09:01:36,960 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.Compactor(225): Compacting 174ce5669bac443c8a3a58c446e4e80c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1732093293076 2024-11-20T09:01:36,960 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.Compactor(225): Compacting aabf66da690a4c57a09c79d93bc18a90, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1732093294636 2024-11-20T09:01:36,961 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e765ef86c58446bc8e368358023e83a1/B of e765ef86c58446bc8e368358023e83a1 into cf9f2ce06d4842a68c7b3dcd66beb210(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T09:01:36,961 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e765ef86c58446bc8e368358023e83a1: 2024-11-20T09:01:36,961 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., storeName=e765ef86c58446bc8e368358023e83a1/B, priority=13, startTime=1732093296526; duration=0sec 2024-11-20T09:01:36,962 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T09:01:36,962 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e765ef86c58446bc8e368358023e83a1:B 2024-11-20T09:01:36,967 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e765ef86c58446bc8e368358023e83a1#C#compaction#446 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T09:01:36,968 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/C/a43a3b186a5941b6b71fb3a4475a18fe is 50, key is test_row_0/C:col10/1732093295279/Put/seqid=0 2024-11-20T09:01:36,971 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742348_1524 (size=12765) 2024-11-20T09:01:36,998 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:36,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52884 deadline: 1732093356989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:36,999 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:36,999 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:36,999 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:01:37,065 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=269 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/A/e52ac2ecf4bd4820823c83c6b97a4e3e 2024-11-20T09:01:37,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/B/489cff995f8f4800b30c244b30bbadb2 is 50, key is test_row_0/B:col10/1732093295314/Put/seqid=0 2024-11-20T09:01:37,075 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742349_1525 (size=12301) 2024-11-20T09:01:37,113 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:37,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52884 deadline: 1732093357107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:37,113 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:37,113 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:37,113 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:01:37,323 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:37,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52884 deadline: 1732093357317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:37,323 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:37,323 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:37,323 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:01:37,375 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/C/a43a3b186a5941b6b71fb3a4475a18fe as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/C/a43a3b186a5941b6b71fb3a4475a18fe 2024-11-20T09:01:37,379 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e765ef86c58446bc8e368358023e83a1/C of e765ef86c58446bc8e368358023e83a1 into a43a3b186a5941b6b71fb3a4475a18fe(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T09:01:37,379 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e765ef86c58446bc8e368358023e83a1: 2024-11-20T09:01:37,379 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., storeName=e765ef86c58446bc8e368358023e83a1/C, priority=13, startTime=1732093296526; duration=0sec 2024-11-20T09:01:37,379 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T09:01:37,379 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e765ef86c58446bc8e368358023e83a1:C 2024-11-20T09:01:37,476 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=269 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/B/489cff995f8f4800b30c244b30bbadb2 2024-11-20T09:01:37,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/C/52bc9be5a58d4423800e6269c8c83487 is 50, key is test_row_0/C:col10/1732093295314/Put/seqid=0 2024-11-20T09:01:37,485 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742350_1526 (size=12301) 2024-11-20T09:01:37,532 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:37,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52840 deadline: 1732093357527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:37,533 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:37,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52854 deadline: 1732093357527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:37,533 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:37,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52804 deadline: 1732093357527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:37,533 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to be1eb2620e0e,34223,1732093159960 for 'test_row_1' in e765ef86c58446bc8e368358023e83a1 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2207 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-20T09:01:37,533 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:37,533 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:37,533 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to be1eb2620e0e,34223,1732093159960 for 'test_row_0' in e765ef86c58446bc8e368358023e83a1 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2206 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-20T09:01:37,533 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:01:37,533 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:37,533 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to be1eb2620e0e,34223,1732093159960 for 'test_row_1' in e765ef86c58446bc8e368358023e83a1 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2203 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-20T09:01:37,534 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:37,534 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:01:37,534 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:37,534 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:37,534 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:01:37,634 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:37,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52884 deadline: 1732093357627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:37,634 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:37,635 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:37,635 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:01:37,886 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=269 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/C/52bc9be5a58d4423800e6269c8c83487 2024-11-20T09:01:37,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/A/e52ac2ecf4bd4820823c83c6b97a4e3e as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/A/e52ac2ecf4bd4820823c83c6b97a4e3e 2024-11-20T09:01:37,892 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/A/e52ac2ecf4bd4820823c83c6b97a4e3e, entries=150, sequenceid=269, filesize=12.0 K 2024-11-20T09:01:37,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/B/489cff995f8f4800b30c244b30bbadb2 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/B/489cff995f8f4800b30c244b30bbadb2 2024-11-20T09:01:37,896 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/B/489cff995f8f4800b30c244b30bbadb2, entries=150, sequenceid=269, filesize=12.0 K 2024-11-20T09:01:37,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/C/52bc9be5a58d4423800e6269c8c83487 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/C/52bc9be5a58d4423800e6269c8c83487 2024-11-20T09:01:37,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-20T09:01:37,899 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/C/52bc9be5a58d4423800e6269c8c83487, entries=150, sequenceid=269, filesize=12.0 K 2024-11-20T09:01:37,900 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for e765ef86c58446bc8e368358023e83a1 in 1244ms, sequenceid=269, compaction requested=false 2024-11-20T09:01:37,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2603): Flush status journal for e765ef86c58446bc8e368358023e83a1: 2024-11-20T09:01:37,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1. 2024-11-20T09:01:37,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=126 2024-11-20T09:01:37,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.HMaster(4169): Remote procedure done, pid=126 2024-11-20T09:01:37,903 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=126, resume processing ppid=125 2024-11-20T09:01:37,903 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=126, ppid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1630 sec 2024-11-20T09:01:37,905 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees in 2.1670 sec 2024-11-20T09:01:38,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(8855): Flush requested on e765ef86c58446bc8e368358023e83a1 2024-11-20T09:01:38,150 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T09:01:38,151 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e765ef86c58446bc8e368358023e83a1 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-20T09:01:38,155 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/A/16c959a1e6b744608527ad48d5c9f7df is 50, key is test_row_0/A:col10/1732093296917/Put/seqid=0 2024-11-20T09:01:38,160 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742351_1527 (size=14741) 2024-11-20T09:01:38,243 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:38,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52884 deadline: 1732093358234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:38,243 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:38,243 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:38,243 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:01:38,359 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:38,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52884 deadline: 1732093358346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:38,359 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:38,359 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:38,359 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:01:38,561 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=291 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/A/16c959a1e6b744608527ad48d5c9f7df 2024-11-20T09:01:38,568 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/B/b4ff984fcc284d62bf4c4330d5998b89 is 50, key is test_row_0/B:col10/1732093296917/Put/seqid=0 2024-11-20T09:01:38,571 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742352_1528 (size=12301) 2024-11-20T09:01:38,577 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:38,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52884 deadline: 1732093358567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:38,577 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:38,577 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:38,577 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:01:38,891 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:38,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52884 deadline: 1732093358886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:38,892 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:38,892 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:38,892 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:01:38,971 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=291 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/B/b4ff984fcc284d62bf4c4330d5998b89 2024-11-20T09:01:38,978 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/C/b6096eea23224fd693fb846b7bb820d2 is 50, key is test_row_0/C:col10/1732093296917/Put/seqid=0 2024-11-20T09:01:38,980 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742353_1529 (size=12301) 2024-11-20T09:01:39,382 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=291 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/C/b6096eea23224fd693fb846b7bb820d2 2024-11-20T09:01:39,386 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/A/16c959a1e6b744608527ad48d5c9f7df as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/A/16c959a1e6b744608527ad48d5c9f7df 2024-11-20T09:01:39,389 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/A/16c959a1e6b744608527ad48d5c9f7df, entries=200, sequenceid=291, filesize=14.4 K 2024-11-20T09:01:39,390 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/B/b4ff984fcc284d62bf4c4330d5998b89 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/B/b4ff984fcc284d62bf4c4330d5998b89 2024-11-20T09:01:39,393 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/B/b4ff984fcc284d62bf4c4330d5998b89, entries=150, sequenceid=291, filesize=12.0 K 2024-11-20T09:01:39,393 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/C/b6096eea23224fd693fb846b7bb820d2 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/C/b6096eea23224fd693fb846b7bb820d2 2024-11-20T09:01:39,396 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/C/b6096eea23224fd693fb846b7bb820d2, entries=150, sequenceid=291, filesize=12.0 K 2024-11-20T09:01:39,397 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=93.93 KB/96180 for e765ef86c58446bc8e368358023e83a1 in 1246ms, sequenceid=291, compaction requested=true 2024-11-20T09:01:39,397 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e765ef86c58446bc8e368358023e83a1: 2024-11-20T09:01:39,397 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e765ef86c58446bc8e368358023e83a1:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T09:01:39,397 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T09:01:39,397 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e765ef86c58446bc8e368358023e83a1:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T09:01:39,397 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T09:01:39,397 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T09:01:39,397 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e765ef86c58446bc8e368358023e83a1:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T09:01:39,397 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T09:01:39,397 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T09:01:39,398 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39807 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T09:01:39,398 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37367 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T09:01:39,398 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HStore(1541): e765ef86c58446bc8e368358023e83a1/A is initiating minor compaction (all files) 2024-11-20T09:01:39,398 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1541): e765ef86c58446bc8e368358023e83a1/B is initiating minor compaction (all files) 2024-11-20T09:01:39,398 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e765ef86c58446bc8e368358023e83a1/A in TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1. 2024-11-20T09:01:39,398 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e765ef86c58446bc8e368358023e83a1/B in TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1. 2024-11-20T09:01:39,398 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/A/f2f72fc1bd5945ea81c3160a0b220707, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/A/e52ac2ecf4bd4820823c83c6b97a4e3e, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/A/16c959a1e6b744608527ad48d5c9f7df] into tmpdir=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp, totalSize=38.9 K 2024-11-20T09:01:39,398 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/B/cf9f2ce06d4842a68c7b3dcd66beb210, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/B/489cff995f8f4800b30c244b30bbadb2, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/B/b4ff984fcc284d62bf4c4330d5998b89] into tmpdir=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp, totalSize=36.5 K 2024-11-20T09:01:39,398 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.Compactor(225): Compacting f2f72fc1bd5945ea81c3160a0b220707, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1732093294636 2024-11-20T09:01:39,398 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting cf9f2ce06d4842a68c7b3dcd66beb210, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1732093294636 2024-11-20T09:01:39,399 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting 489cff995f8f4800b30c244b30bbadb2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=269, earliestPutTs=1732093295314 2024-11-20T09:01:39,399 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.Compactor(225): Compacting e52ac2ecf4bd4820823c83c6b97a4e3e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=269, earliestPutTs=1732093295314 2024-11-20T09:01:39,399 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting b4ff984fcc284d62bf4c4330d5998b89, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1732093296917 2024-11-20T09:01:39,399 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.Compactor(225): Compacting 16c959a1e6b744608527ad48d5c9f7df, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1732093296917 2024-11-20T09:01:39,410 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e765ef86c58446bc8e368358023e83a1#B#compaction#452 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-20T09:01:39,410 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e765ef86c58446bc8e368358023e83a1#A#compaction#453 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T09:01:39,411 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/B/642e16184be0436ea1fed4ca81af525e is 50, key is test_row_0/B:col10/1732093296917/Put/seqid=0 2024-11-20T09:01:39,411 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/A/5b93cf58f8594e35914ec9c4bba4637f is 50, key is test_row_0/A:col10/1732093296917/Put/seqid=0 2024-11-20T09:01:39,412 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T09:01:39,412 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e765ef86c58446bc8e368358023e83a1 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-20T09:01:39,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(8855): Flush requested on e765ef86c58446bc8e368358023e83a1 2024-11-20T09:01:39,417 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742354_1530 (size=13017) 2024-11-20T09:01:39,422 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742355_1531 (size=13017) 2024-11-20T09:01:39,425 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/A/590d4880903248979585284bf2dab80e is 50, key is test_row_0/A:col10/1732093299410/Put/seqid=0 2024-11-20T09:01:39,429 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742356_1532 (size=14741) 2024-11-20T09:01:39,548 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:39,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52884 deadline: 1732093359542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:39,549 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:39,549 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:39,549 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:01:39,554 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:39,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52854 deadline: 1732093359546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:39,554 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:39,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52804 deadline: 1732093359547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:39,554 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to be1eb2620e0e,34223,1732093159960 for 'test_row_0' in e765ef86c58446bc8e368358023e83a1 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4228 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-20T09:01:39,555 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:39,555 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:39,555 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to be1eb2620e0e,34223,1732093159960 for 'test_row_1' in e765ef86c58446bc8e368358023e83a1 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4224 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-20T09:01:39,555 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:01:39,555 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:39,555 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:39,555 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:01:39,562 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:39,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52840 deadline: 1732093359549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:39,562 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:39,562 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:39,562 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:01:39,663 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:39,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52884 deadline: 1732093359657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:39,663 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:39,663 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:39,663 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:01:39,676 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:39,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52840 deadline: 1732093359667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:39,676 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:39,676 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:39,676 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:01:39,821 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/B/642e16184be0436ea1fed4ca81af525e as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/B/642e16184be0436ea1fed4ca81af525e 2024-11-20T09:01:39,830 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=309 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/A/590d4880903248979585284bf2dab80e 2024-11-20T09:01:39,830 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e765ef86c58446bc8e368358023e83a1/B of e765ef86c58446bc8e368358023e83a1 into 642e16184be0436ea1fed4ca81af525e(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T09:01:39,830 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e765ef86c58446bc8e368358023e83a1: 2024-11-20T09:01:39,830 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., storeName=e765ef86c58446bc8e368358023e83a1/B, priority=13, startTime=1732093299397; duration=0sec 2024-11-20T09:01:39,831 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T09:01:39,831 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e765ef86c58446bc8e368358023e83a1:B 2024-11-20T09:01:39,831 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T09:01:39,832 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/A/5b93cf58f8594e35914ec9c4bba4637f as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/A/5b93cf58f8594e35914ec9c4bba4637f 2024-11-20T09:01:39,832 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37367 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T09:01:39,832 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1541): e765ef86c58446bc8e368358023e83a1/C is initiating minor compaction (all files) 2024-11-20T09:01:39,832 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e765ef86c58446bc8e368358023e83a1/C in TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1. 2024-11-20T09:01:39,832 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/C/a43a3b186a5941b6b71fb3a4475a18fe, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/C/52bc9be5a58d4423800e6269c8c83487, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/C/b6096eea23224fd693fb846b7bb820d2] into tmpdir=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp, totalSize=36.5 K 2024-11-20T09:01:39,833 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting a43a3b186a5941b6b71fb3a4475a18fe, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1732093294636 2024-11-20T09:01:39,834 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting 52bc9be5a58d4423800e6269c8c83487, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=269, earliestPutTs=1732093295314 2024-11-20T09:01:39,835 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting b6096eea23224fd693fb846b7bb820d2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1732093296917 2024-11-20T09:01:39,837 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/B/9f01e0eaf92b4b8cb73ad682d7a024b2 is 50, key is test_row_0/B:col10/1732093299410/Put/seqid=0 2024-11-20T09:01:39,837 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e765ef86c58446bc8e368358023e83a1/A of e765ef86c58446bc8e368358023e83a1 into 5b93cf58f8594e35914ec9c4bba4637f(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T09:01:39,838 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e765ef86c58446bc8e368358023e83a1: 2024-11-20T09:01:39,838 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., storeName=e765ef86c58446bc8e368358023e83a1/A, priority=13, startTime=1732093299397; duration=0sec 2024-11-20T09:01:39,838 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T09:01:39,838 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e765ef86c58446bc8e368358023e83a1:A 2024-11-20T09:01:39,847 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742357_1533 (size=12301) 2024-11-20T09:01:39,848 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e765ef86c58446bc8e368358023e83a1#C#compaction#456 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T09:01:39,848 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/C/c627a7cf687e425cad6d964ed0272c35 is 50, key is test_row_0/C:col10/1732093296917/Put/seqid=0 2024-11-20T09:01:39,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742358_1534 (size=13017) 2024-11-20T09:01:39,877 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:39,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52884 deadline: 1732093359867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:39,877 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:39,877 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:39,877 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:01:39,893 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:39,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52840 deadline: 1732093359886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:39,893 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:39,894 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:39,894 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:01:39,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-20T09:01:39,907 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T09:01:39,908 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T09:01:39,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] procedure2.ProcedureExecutor(1139): Stored pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees 2024-11-20T09:01:39,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-20T09:01:39,909 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T09:01:39,910 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T09:01:39,910 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=128, ppid=127, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T09:01:40,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-20T09:01:40,062 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-20T09:01:40,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1. 2024-11-20T09:01:40,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1. as already flushing 2024-11-20T09:01:40,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1. 2024-11-20T09:01:40,063 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => e765ef86c58446bc8e368358023e83a1, NAME => 'TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:40,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => e765ef86c58446bc8e368358023e83a1, NAME => 'TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:40,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e765ef86c58446bc8e368358023e83a1, NAME => 'TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e765ef86c58446bc8e368358023e83a1, NAME => 'TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:40,195 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:40,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52884 deadline: 1732093360187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:40,195 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:40,196 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:40,196 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:01:40,208 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:40,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52840 deadline: 1732093360197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:40,209 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:40,209 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:40,209 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:01:40,215 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-20T09:01:40,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1. 2024-11-20T09:01:40,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1. as already flushing 2024-11-20T09:01:40,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1. 2024-11-20T09:01:40,215 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => e765ef86c58446bc8e368358023e83a1, NAME => 'TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:40,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => e765ef86c58446bc8e368358023e83a1, NAME => 'TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:40,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e765ef86c58446bc8e368358023e83a1, NAME => 'TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e765ef86c58446bc8e368358023e83a1, NAME => 'TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:40,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-20T09:01:40,247 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=309 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/B/9f01e0eaf92b4b8cb73ad682d7a024b2 2024-11-20T09:01:40,254 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/C/d1aba9fb1c974dc9a10390119e94efde is 50, key is test_row_0/C:col10/1732093299410/Put/seqid=0 2024-11-20T09:01:40,257 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/C/c627a7cf687e425cad6d964ed0272c35 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/C/c627a7cf687e425cad6d964ed0272c35 2024-11-20T09:01:40,259 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742359_1535 (size=12301) 2024-11-20T09:01:40,262 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e765ef86c58446bc8e368358023e83a1/C of e765ef86c58446bc8e368358023e83a1 into c627a7cf687e425cad6d964ed0272c35(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T09:01:40,262 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e765ef86c58446bc8e368358023e83a1: 2024-11-20T09:01:40,262 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., storeName=e765ef86c58446bc8e368358023e83a1/C, priority=13, startTime=1732093299397; duration=0sec 2024-11-20T09:01:40,262 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T09:01:40,262 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e765ef86c58446bc8e368358023e83a1:C 2024-11-20T09:01:40,368 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-20T09:01:40,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1. 2024-11-20T09:01:40,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1. as already flushing 2024-11-20T09:01:40,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1. 2024-11-20T09:01:40,369 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => e765ef86c58446bc8e368358023e83a1, NAME => 'TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:40,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => e765ef86c58446bc8e368358023e83a1, NAME => 'TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:40,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e765ef86c58446bc8e368358023e83a1, NAME => 'TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e765ef86c58446bc8e368358023e83a1, NAME => 'TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:40,522 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-20T09:01:40,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1. 2024-11-20T09:01:40,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1. as already flushing 2024-11-20T09:01:40,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1. 2024-11-20T09:01:40,522 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => e765ef86c58446bc8e368358023e83a1, NAME => 'TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:40,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => e765ef86c58446bc8e368358023e83a1, NAME => 'TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:40,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e765ef86c58446bc8e368358023e83a1, NAME => 'TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e765ef86c58446bc8e368358023e83a1, NAME => 'TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:40,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-20T09:01:40,660 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=309 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/C/d1aba9fb1c974dc9a10390119e94efde 2024-11-20T09:01:40,663 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/A/590d4880903248979585284bf2dab80e as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/A/590d4880903248979585284bf2dab80e 2024-11-20T09:01:40,667 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/A/590d4880903248979585284bf2dab80e, entries=200, sequenceid=309, filesize=14.4 K 2024-11-20T09:01:40,667 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/B/9f01e0eaf92b4b8cb73ad682d7a024b2 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/B/9f01e0eaf92b4b8cb73ad682d7a024b2 2024-11-20T09:01:40,670 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/B/9f01e0eaf92b4b8cb73ad682d7a024b2, entries=150, sequenceid=309, filesize=12.0 K 2024-11-20T09:01:40,671 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/C/d1aba9fb1c974dc9a10390119e94efde as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/C/d1aba9fb1c974dc9a10390119e94efde 2024-11-20T09:01:40,673 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/C/d1aba9fb1c974dc9a10390119e94efde, entries=150, sequenceid=309, filesize=12.0 K 2024-11-20T09:01:40,674 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for e765ef86c58446bc8e368358023e83a1 in 1262ms, sequenceid=309, compaction requested=false 2024-11-20T09:01:40,675 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e765ef86c58446bc8e368358023e83a1: 2024-11-20T09:01:40,675 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-20T09:01:40,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1. 2024-11-20T09:01:40,675 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2902): Flushing e765ef86c58446bc8e368358023e83a1 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-20T09:01:40,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/A/72d7dac81ace448095c1e9ab3949b16d is 50, key is test_row_0/A:col10/1732093299540/Put/seqid=0 2024-11-20T09:01:40,683 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742360_1536 (size=12301) 2024-11-20T09:01:40,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(8855): Flush requested on e765ef86c58446bc8e368358023e83a1 2024-11-20T09:01:40,713 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1. as already flushing 2024-11-20T09:01:40,784 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:40,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52884 deadline: 1732093360773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:40,784 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:40,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52840 deadline: 1732093360783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:40,785 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:40,785 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:40,785 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:01:40,785 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:40,785 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:40,785 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:01:40,893 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:40,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52884 deadline: 1732093360887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:40,893 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:40,893 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:40,893 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:40,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52840 deadline: 1732093360887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:40,893 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:01:40,893 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:40,893 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:40,894 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:01:41,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-20T09:01:41,084 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=330 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/A/72d7dac81ace448095c1e9ab3949b16d 2024-11-20T09:01:41,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/B/e661acf1781c4c75863417a68efeb780 is 50, key is test_row_0/B:col10/1732093299540/Put/seqid=0 2024-11-20T09:01:41,094 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742361_1537 (size=12301) 2024-11-20T09:01:41,100 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:41,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52884 deadline: 1732093361096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:41,101 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:41,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52840 deadline: 1732093361097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:41,101 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:41,101 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:41,101 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:01:41,101 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:41,101 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:41,101 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:01:41,416 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:41,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52884 deadline: 1732093361407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:41,416 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:41,416 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:41,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52840 deadline: 1732093361407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:41,416 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:41,416 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:01:41,417 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:41,417 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:41,417 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:01:41,495 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=330 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/B/e661acf1781c4c75863417a68efeb780 2024-11-20T09:01:41,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/C/6f48b3fabfe44e90aa01b6e8213502dd is 50, key is test_row_0/C:col10/1732093299540/Put/seqid=0 2024-11-20T09:01:41,505 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742362_1538 (size=12301) 2024-11-20T09:01:41,905 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=330 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/C/6f48b3fabfe44e90aa01b6e8213502dd 2024-11-20T09:01:41,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/A/72d7dac81ace448095c1e9ab3949b16d as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/A/72d7dac81ace448095c1e9ab3949b16d 2024-11-20T09:01:41,912 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/A/72d7dac81ace448095c1e9ab3949b16d, entries=150, sequenceid=330, filesize=12.0 K 2024-11-20T09:01:41,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/B/e661acf1781c4c75863417a68efeb780 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/B/e661acf1781c4c75863417a68efeb780 2024-11-20T09:01:41,915 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/B/e661acf1781c4c75863417a68efeb780, entries=150, sequenceid=330, filesize=12.0 K 2024-11-20T09:01:41,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/C/6f48b3fabfe44e90aa01b6e8213502dd as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/C/6f48b3fabfe44e90aa01b6e8213502dd 2024-11-20T09:01:41,919 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/C/6f48b3fabfe44e90aa01b6e8213502dd, entries=150, sequenceid=330, filesize=12.0 K 2024-11-20T09:01:41,920 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for e765ef86c58446bc8e368358023e83a1 in 1244ms, sequenceid=330, compaction requested=true 2024-11-20T09:01:41,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2603): Flush status journal for e765ef86c58446bc8e368358023e83a1: 2024-11-20T09:01:41,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1. 2024-11-20T09:01:41,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=128 2024-11-20T09:01:41,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.HMaster(4169): Remote procedure done, pid=128 2024-11-20T09:01:41,922 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=128, resume processing ppid=127 2024-11-20T09:01:41,923 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=128, ppid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0110 sec 2024-11-20T09:01:41,924 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees in 2.0150 sec 2024-11-20T09:01:41,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(8855): Flush requested on e765ef86c58446bc8e368358023e83a1 2024-11-20T09:01:41,933 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T09:01:41,933 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e765ef86c58446bc8e368358023e83a1 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-20T09:01:41,937 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/A/21a5d27bb00444e2be3dd3439cd1095b is 50, key is test_row_0/A:col10/1732093300769/Put/seqid=0 2024-11-20T09:01:41,940 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742363_1539 (size=14741) 2024-11-20T09:01:41,989 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:41,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52840 deadline: 1732093361986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:41,989 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:41,989 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:41,990 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:01:41,997 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:41,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52884 deadline: 1732093361987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:41,997 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:41,997 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:41,997 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:01:42,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-20T09:01:42,057 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T09:01:42,058 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T09:01:42,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] procedure2.ProcedureExecutor(1139): Stored pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees 2024-11-20T09:01:42,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-20T09:01:42,059 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T09:01:42,060 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T09:01:42,060 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=130, ppid=129, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T09:01:42,104 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:42,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52840 deadline: 1732093362097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:42,104 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:42,104 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:42,104 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:01:42,115 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:42,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52884 deadline: 1732093362106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:42,116 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:42,116 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:42,116 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:01:42,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-20T09:01:42,212 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-20T09:01:42,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1. 2024-11-20T09:01:42,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1. as already flushing 2024-11-20T09:01:42,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1. 2024-11-20T09:01:42,213 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => e765ef86c58446bc8e368358023e83a1, NAME => 'TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:42,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => e765ef86c58446bc8e368358023e83a1, NAME => 'TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:42,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e765ef86c58446bc8e368358023e83a1, NAME => 'TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e765ef86c58446bc8e368358023e83a1, NAME => 'TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:42,310 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:42,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52840 deadline: 1732093362307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:42,310 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:42,310 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:42,310 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:01:42,334 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:42,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52884 deadline: 1732093362326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:42,334 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:42,334 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:42,334 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:01:42,341 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=349 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/A/21a5d27bb00444e2be3dd3439cd1095b 2024-11-20T09:01:42,347 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/B/68518345671e41218a48b5495b49e296 is 50, key is test_row_0/B:col10/1732093300769/Put/seqid=0 2024-11-20T09:01:42,353 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742364_1540 (size=12301) 2024-11-20T09:01:42,365 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-20T09:01:42,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1. 2024-11-20T09:01:42,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1. as already flushing 2024-11-20T09:01:42,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1. 2024-11-20T09:01:42,365 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => e765ef86c58446bc8e368358023e83a1, NAME => 'TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:42,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => e765ef86c58446bc8e368358023e83a1, NAME => 'TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:42,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e765ef86c58446bc8e368358023e83a1, NAME => 'TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e765ef86c58446bc8e368358023e83a1, NAME => 'TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:42,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-20T09:01:42,518 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-20T09:01:42,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1. 2024-11-20T09:01:42,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1. as already flushing 2024-11-20T09:01:42,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1. 2024-11-20T09:01:42,518 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => e765ef86c58446bc8e368358023e83a1, NAME => 'TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:42,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => e765ef86c58446bc8e368358023e83a1, NAME => 'TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:42,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e765ef86c58446bc8e368358023e83a1, NAME => 'TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e765ef86c58446bc8e368358023e83a1, NAME => 'TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:42,624 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:42,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52840 deadline: 1732093362617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:42,624 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:42,624 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:42,624 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:01:42,656 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:42,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52884 deadline: 1732093362647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:42,657 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:42,657 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:42,657 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:01:42,670 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-20T09:01:42,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1. 2024-11-20T09:01:42,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1. as already flushing 2024-11-20T09:01:42,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1. 2024-11-20T09:01:42,671 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => e765ef86c58446bc8e368358023e83a1, NAME => 'TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:42,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => e765ef86c58446bc8e368358023e83a1, NAME => 'TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:42,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e765ef86c58446bc8e368358023e83a1, NAME => 'TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e765ef86c58446bc8e368358023e83a1, NAME => 'TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:42,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-20T09:01:42,754 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=349 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/B/68518345671e41218a48b5495b49e296 2024-11-20T09:01:42,760 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/C/f2f062464cc64f98b2a617e7ad21599c is 50, key is test_row_0/C:col10/1732093300769/Put/seqid=0 2024-11-20T09:01:42,763 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742365_1541 (size=12301) 2024-11-20T09:01:42,823 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-20T09:01:42,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1. 2024-11-20T09:01:42,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1. as already flushing 2024-11-20T09:01:42,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1. 2024-11-20T09:01:42,824 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => e765ef86c58446bc8e368358023e83a1, NAME => 'TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:42,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => e765ef86c58446bc8e368358023e83a1, NAME => 'TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:42,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e765ef86c58446bc8e368358023e83a1, NAME => 'TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e765ef86c58446bc8e368358023e83a1, NAME => 'TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:42,977 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-20T09:01:42,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1. 2024-11-20T09:01:42,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1. as already flushing 2024-11-20T09:01:42,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1. 2024-11-20T09:01:42,977 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => e765ef86c58446bc8e368358023e83a1, NAME => 'TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:42,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => e765ef86c58446bc8e368358023e83a1, NAME => 'TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:42,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e765ef86c58446bc8e368358023e83a1, NAME => 'TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e765ef86c58446bc8e368358023e83a1, NAME => 'TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:43,130 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-20T09:01:43,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1. 2024-11-20T09:01:43,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1. as already flushing 2024-11-20T09:01:43,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1. 2024-11-20T09:01:43,130 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => e765ef86c58446bc8e368358023e83a1, NAME => 'TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:43,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => e765ef86c58446bc8e368358023e83a1, NAME => 'TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:43,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e765ef86c58446bc8e368358023e83a1, NAME => 'TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e765ef86c58446bc8e368358023e83a1, NAME => 'TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:43,144 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:43,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52840 deadline: 1732093363137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:43,144 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:43,144 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:43,144 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:01:43,164 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=349 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/C/f2f062464cc64f98b2a617e7ad21599c 2024-11-20T09:01:43,168 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/A/21a5d27bb00444e2be3dd3439cd1095b as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/A/21a5d27bb00444e2be3dd3439cd1095b 2024-11-20T09:01:43,171 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/A/21a5d27bb00444e2be3dd3439cd1095b, entries=200, sequenceid=349, filesize=14.4 K 2024-11-20T09:01:43,171 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/B/68518345671e41218a48b5495b49e296 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/B/68518345671e41218a48b5495b49e296 2024-11-20T09:01:43,174 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/B/68518345671e41218a48b5495b49e296, entries=150, sequenceid=349, filesize=12.0 K 2024-11-20T09:01:43,175 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/C/f2f062464cc64f98b2a617e7ad21599c as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/C/f2f062464cc64f98b2a617e7ad21599c 2024-11-20T09:01:43,177 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:43,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52884 deadline: 1732093363167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:43,177 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:43,177 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:43,177 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:01:43,178 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/C/f2f062464cc64f98b2a617e7ad21599c, entries=150, sequenceid=349, filesize=12.0 K 2024-11-20T09:01:43,179 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=93.93 KB/96180 for e765ef86c58446bc8e368358023e83a1 in 1246ms, sequenceid=349, compaction requested=true 2024-11-20T09:01:43,179 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e765ef86c58446bc8e368358023e83a1: 2024-11-20T09:01:43,179 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e765ef86c58446bc8e368358023e83a1:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T09:01:43,179 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T09:01:43,179 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T09:01:43,179 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e765ef86c58446bc8e368358023e83a1:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T09:01:43,179 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T09:01:43,179 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T09:01:43,179 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e765ef86c58446bc8e368358023e83a1:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T09:01:43,179 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T09:01:43,180 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 54800 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T09:01:43,180 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49920 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T09:01:43,180 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1541): e765ef86c58446bc8e368358023e83a1/B is initiating minor compaction (all files) 2024-11-20T09:01:43,180 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HStore(1541): e765ef86c58446bc8e368358023e83a1/A is initiating minor compaction (all files) 2024-11-20T09:01:43,180 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e765ef86c58446bc8e368358023e83a1/B in TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1. 2024-11-20T09:01:43,180 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e765ef86c58446bc8e368358023e83a1/A in TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1. 2024-11-20T09:01:43,180 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/A/5b93cf58f8594e35914ec9c4bba4637f, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/A/590d4880903248979585284bf2dab80e, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/A/72d7dac81ace448095c1e9ab3949b16d, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/A/21a5d27bb00444e2be3dd3439cd1095b] into tmpdir=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp, totalSize=53.5 K 2024-11-20T09:01:43,180 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/B/642e16184be0436ea1fed4ca81af525e, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/B/9f01e0eaf92b4b8cb73ad682d7a024b2, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/B/e661acf1781c4c75863417a68efeb780, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/B/68518345671e41218a48b5495b49e296] into tmpdir=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp, totalSize=48.8 K 2024-11-20T09:01:43,181 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting 642e16184be0436ea1fed4ca81af525e, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1732093296917 2024-11-20T09:01:43,181 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5b93cf58f8594e35914ec9c4bba4637f, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1732093296917 2024-11-20T09:01:43,181 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.Compactor(225): Compacting 590d4880903248979585284bf2dab80e, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1732093298170 2024-11-20T09:01:43,181 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting 9f01e0eaf92b4b8cb73ad682d7a024b2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1732093298170 2024-11-20T09:01:43,181 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting e661acf1781c4c75863417a68efeb780, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=330, earliestPutTs=1732093299501 2024-11-20T09:01:43,181 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.Compactor(225): Compacting 72d7dac81ace448095c1e9ab3949b16d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=330, earliestPutTs=1732093299501 2024-11-20T09:01:43,181 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting 68518345671e41218a48b5495b49e296, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=349, earliestPutTs=1732093300769 2024-11-20T09:01:43,182 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.Compactor(225): Compacting 21a5d27bb00444e2be3dd3439cd1095b, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=349, earliestPutTs=1732093300743 2024-11-20T09:01:43,190 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e765ef86c58446bc8e368358023e83a1#B#compaction#464 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T09:01:43,190 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/B/cd613d42ffda4d22994af7647f69ae48 is 50, key is test_row_0/B:col10/1732093300769/Put/seqid=0 2024-11-20T09:01:43,193 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e765ef86c58446bc8e368358023e83a1#A#compaction#465 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T09:01:43,194 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/A/70ca48fde567446cae66819396c5cd18 is 50, key is test_row_0/A:col10/1732093300769/Put/seqid=0 2024-11-20T09:01:43,196 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742366_1542 (size=13153) 2024-11-20T09:01:43,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-20T09:01:43,198 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742367_1543 (size=13153) 2024-11-20T09:01:43,283 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-20T09:01:43,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1. 2024-11-20T09:01:43,283 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2902): Flushing e765ef86c58446bc8e368358023e83a1 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-20T09:01:43,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/A/493a3247619c4385a5b8b402d784bf60 is 50, key is test_row_0/A:col10/1732093301974/Put/seqid=0 2024-11-20T09:01:43,291 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742368_1544 (size=12301) 2024-11-20T09:01:43,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(8855): Flush requested on e765ef86c58446bc8e368358023e83a1 2024-11-20T09:01:43,593 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1. as already flushing 2024-11-20T09:01:43,599 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/B/cd613d42ffda4d22994af7647f69ae48 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/B/cd613d42ffda4d22994af7647f69ae48 2024-11-20T09:01:43,603 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/A/70ca48fde567446cae66819396c5cd18 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/A/70ca48fde567446cae66819396c5cd18 2024-11-20T09:01:43,605 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e765ef86c58446bc8e368358023e83a1/B of e765ef86c58446bc8e368358023e83a1 into cd613d42ffda4d22994af7647f69ae48(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T09:01:43,605 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e765ef86c58446bc8e368358023e83a1: 2024-11-20T09:01:43,605 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., storeName=e765ef86c58446bc8e368358023e83a1/B, priority=12, startTime=1732093303179; duration=0sec 2024-11-20T09:01:43,605 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T09:01:43,605 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e765ef86c58446bc8e368358023e83a1:B 2024-11-20T09:01:43,605 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T09:01:43,607 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49920 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T09:01:43,607 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1541): e765ef86c58446bc8e368358023e83a1/C is initiating minor compaction (all files) 2024-11-20T09:01:43,607 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e765ef86c58446bc8e368358023e83a1/C in TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1. 2024-11-20T09:01:43,607 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/C/c627a7cf687e425cad6d964ed0272c35, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/C/d1aba9fb1c974dc9a10390119e94efde, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/C/6f48b3fabfe44e90aa01b6e8213502dd, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/C/f2f062464cc64f98b2a617e7ad21599c] into tmpdir=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp, totalSize=48.8 K 2024-11-20T09:01:43,608 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting c627a7cf687e425cad6d964ed0272c35, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1732093296917 2024-11-20T09:01:43,608 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting d1aba9fb1c974dc9a10390119e94efde, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1732093298170 2024-11-20T09:01:43,610 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e765ef86c58446bc8e368358023e83a1/A of e765ef86c58446bc8e368358023e83a1 into 70ca48fde567446cae66819396c5cd18(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T09:01:43,610 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e765ef86c58446bc8e368358023e83a1: 2024-11-20T09:01:43,610 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., storeName=e765ef86c58446bc8e368358023e83a1/A, priority=12, startTime=1732093303179; duration=0sec 2024-11-20T09:01:43,610 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T09:01:43,610 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e765ef86c58446bc8e368358023e83a1:A 2024-11-20T09:01:43,611 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting 6f48b3fabfe44e90aa01b6e8213502dd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=330, earliestPutTs=1732093299501 2024-11-20T09:01:43,611 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting f2f062464cc64f98b2a617e7ad21599c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=349, earliestPutTs=1732093300769 2024-11-20T09:01:43,617 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e765ef86c58446bc8e368358023e83a1#C#compaction#467 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T09:01:43,618 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/C/977dffe95a934a9c80e9a8f7a2779128 is 50, key is test_row_0/C:col10/1732093300769/Put/seqid=0 2024-11-20T09:01:43,621 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742369_1545 (size=13153) 2024-11-20T09:01:43,692 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=366 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/A/493a3247619c4385a5b8b402d784bf60 2024-11-20T09:01:43,702 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:43,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52854 deadline: 1732093363693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:43,702 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:43,702 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:43,703 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:01:43,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/B/831a3e5cede142d4b7067895861d17b0 is 50, key is test_row_0/B:col10/1732093301974/Put/seqid=0 2024-11-20T09:01:43,707 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:43,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52804 deadline: 1732093363694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:43,707 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:43,707 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:43,707 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:01:43,710 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742370_1546 (size=12301) 2024-11-20T09:01:43,817 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:43,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52854 deadline: 1732093363807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:43,818 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:43,818 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:43,818 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:01:43,822 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:43,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52804 deadline: 1732093363817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:43,822 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:43,822 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:43,822 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:01:43,996 INFO [Thread-2098 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2098. 2024-11-20T09:01:43,996 DEBUG [Thread-2098 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T09:01:43,996 DEBUG [Thread-2098 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T09:01:43,996 DEBUG [Thread-2098 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T09:01:43,996 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T09:01:43,996 INFO [Thread-2094 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2094. 2024-11-20T09:01:43,997 DEBUG [Thread-2094 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T09:01:43,997 DEBUG [Thread-2094 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T09:01:43,997 DEBUG [Thread-2094 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T09:01:43,997 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T09:01:44,004 INFO [Thread-2096 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2096. 2024-11-20T09:01:44,004 DEBUG [Thread-2096 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T09:01:44,004 DEBUG [Thread-2096 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T09:01:44,004 DEBUG [Thread-2096 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T09:01:44,004 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T09:01:44,008 INFO [Thread-2100 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2100. 2024-11-20T09:01:44,008 DEBUG [Thread-2100 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T09:01:44,008 DEBUG [Thread-2100 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T09:01:44,008 INFO [Thread-2102 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2102. 2024-11-20T09:01:44,008 DEBUG [Thread-2102 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T09:01:44,008 DEBUG [Thread-2102 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T09:01:44,008 DEBUG [Thread-2100 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T09:01:44,008 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T09:01:44,008 DEBUG [Thread-2102 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T09:01:44,009 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T09:01:44,025 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/C/977dffe95a934a9c80e9a8f7a2779128 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/C/977dffe95a934a9c80e9a8f7a2779128 2024-11-20T09:01:44,027 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:44,027 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:44,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52854 deadline: 1732093364026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:44,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52804 deadline: 1732093364027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:44,027 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:44,027 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:44,027 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:01:44,027 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:44,027 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:44,027 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:01:44,028 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e765ef86c58446bc8e368358023e83a1/C of e765ef86c58446bc8e368358023e83a1 into 977dffe95a934a9c80e9a8f7a2779128(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T09:01:44,028 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e765ef86c58446bc8e368358023e83a1: 2024-11-20T09:01:44,028 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., storeName=e765ef86c58446bc8e368358023e83a1/C, priority=12, startTime=1732093303179; duration=0sec 2024-11-20T09:01:44,028 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T09:01:44,028 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e765ef86c58446bc8e368358023e83a1:C 2024-11-20T09:01:44,111 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=366 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/B/831a3e5cede142d4b7067895861d17b0 2024-11-20T09:01:44,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/C/2a1459a8998445038f33255c514a5fd2 is 50, key is test_row_0/C:col10/1732093301974/Put/seqid=0 2024-11-20T09:01:44,118 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742371_1547 (size=12301) 2024-11-20T09:01:44,157 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:44,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52840 deadline: 1732093364157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:44,157 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to be1eb2620e0e,34223,1732093159960 for 'test_row_2' in e765ef86c58446bc8e368358023e83a1 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2171 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-20T09:01:44,158 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:44,158 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:44,158 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:01:44,197 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:44,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52884 deadline: 1732093364197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:44,198 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to be1eb2620e0e,34223,1732093159960 for 'test_row_0' in e765ef86c58446bc8e368358023e83a1 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2210 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-20T09:01:44,198 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:44,198 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:44,198 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:01:44,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-20T09:01:44,338 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:44,338 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:44,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52854 deadline: 1732093364337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:44,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52804 deadline: 1732093364337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:44,338 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:44,338 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 , the old value is region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:44,338 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:44,338 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:44,338 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:01:44,338 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., hostname=be1eb2620e0e,34223,1732093159960, seqNum=2 because the exception is null or not the one we care about 2024-11-20T09:01:44,519 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=366 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/C/2a1459a8998445038f33255c514a5fd2 2024-11-20T09:01:44,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/A/493a3247619c4385a5b8b402d784bf60 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/A/493a3247619c4385a5b8b402d784bf60 2024-11-20T09:01:44,525 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/A/493a3247619c4385a5b8b402d784bf60, entries=150, sequenceid=366, filesize=12.0 K 2024-11-20T09:01:44,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/B/831a3e5cede142d4b7067895861d17b0 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/B/831a3e5cede142d4b7067895861d17b0 2024-11-20T09:01:44,528 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/B/831a3e5cede142d4b7067895861d17b0, entries=150, sequenceid=366, filesize=12.0 K 2024-11-20T09:01:44,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/C/2a1459a8998445038f33255c514a5fd2 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/C/2a1459a8998445038f33255c514a5fd2 2024-11-20T09:01:44,531 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/C/2a1459a8998445038f33255c514a5fd2, entries=150, sequenceid=366, filesize=12.0 K 2024-11-20T09:01:44,532 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for e765ef86c58446bc8e368358023e83a1 in 1249ms, sequenceid=366, compaction requested=false 2024-11-20T09:01:44,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2603): Flush status journal for e765ef86c58446bc8e368358023e83a1: 2024-11-20T09:01:44,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1. 2024-11-20T09:01:44,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=130 2024-11-20T09:01:44,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.HMaster(4169): Remote procedure done, pid=130 2024-11-20T09:01:44,535 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=130, resume processing ppid=129 2024-11-20T09:01:44,535 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=130, ppid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4730 sec 2024-11-20T09:01:44,536 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees in 2.4770 sec 2024-11-20T09:01:44,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(8855): Flush requested on e765ef86c58446bc8e368358023e83a1 2024-11-20T09:01:44,848 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T09:01:44,848 INFO [Thread-2085 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2085. 2024-11-20T09:01:44,848 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e765ef86c58446bc8e368358023e83a1 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-20T09:01:44,848 DEBUG [Thread-2085 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T09:01:44,848 DEBUG [Thread-2085 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T09:01:44,849 INFO [Thread-2083 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2083. 2024-11-20T09:01:44,849 DEBUG [Thread-2085 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T09:01:44,849 DEBUG [Thread-2083 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T09:01:44,849 DEBUG [Thread-2083 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T09:01:44,849 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T09:01:44,849 DEBUG [Thread-2083 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T09:01:44,849 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T09:01:44,852 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/A/8dfd75f108464e2ba593fd8fbd6e6625 is 50, key is test_row_0/A:col10/1732093303692/Put/seqid=0 2024-11-20T09:01:44,855 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742372_1548 (size=12301) 2024-11-20T09:01:45,256 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=390 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/A/8dfd75f108464e2ba593fd8fbd6e6625 2024-11-20T09:01:45,261 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/B/d57d1a7799904fb8a183bb0530282a07 is 50, key is test_row_0/B:col10/1732093303692/Put/seqid=0 2024-11-20T09:01:45,264 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742373_1549 (size=12301) 2024-11-20T09:01:45,665 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=390 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/B/d57d1a7799904fb8a183bb0530282a07 2024-11-20T09:01:45,670 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/C/f007d72587354260803eae9ff3d2caee is 50, key is test_row_0/C:col10/1732093303692/Put/seqid=0 2024-11-20T09:01:45,672 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742374_1550 (size=12301) 2024-11-20T09:01:46,073 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=390 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/C/f007d72587354260803eae9ff3d2caee 2024-11-20T09:01:46,077 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/A/8dfd75f108464e2ba593fd8fbd6e6625 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/A/8dfd75f108464e2ba593fd8fbd6e6625 2024-11-20T09:01:46,079 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/A/8dfd75f108464e2ba593fd8fbd6e6625, entries=150, sequenceid=390, filesize=12.0 K 2024-11-20T09:01:46,080 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/B/d57d1a7799904fb8a183bb0530282a07 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/B/d57d1a7799904fb8a183bb0530282a07 2024-11-20T09:01:46,083 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/B/d57d1a7799904fb8a183bb0530282a07, entries=150, sequenceid=390, filesize=12.0 K 2024-11-20T09:01:46,083 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/C/f007d72587354260803eae9ff3d2caee as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/C/f007d72587354260803eae9ff3d2caee 2024-11-20T09:01:46,085 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/C/f007d72587354260803eae9ff3d2caee, entries=150, sequenceid=390, filesize=12.0 K 2024-11-20T09:01:46,086 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=0 B/0 for e765ef86c58446bc8e368358023e83a1 in 1238ms, sequenceid=390, compaction requested=true 2024-11-20T09:01:46,086 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e765ef86c58446bc8e368358023e83a1: 2024-11-20T09:01:46,086 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e765ef86c58446bc8e368358023e83a1:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T09:01:46,086 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T09:01:46,086 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e765ef86c58446bc8e368358023e83a1:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T09:01:46,086 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T09:01:46,086 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e765ef86c58446bc8e368358023e83a1:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T09:01:46,086 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T09:01:46,086 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T09:01:46,087 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T09:01:46,087 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T09:01:46,087 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T09:01:46,087 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1541): e765ef86c58446bc8e368358023e83a1/B is initiating minor compaction (all files) 2024-11-20T09:01:46,087 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HStore(1541): e765ef86c58446bc8e368358023e83a1/A is initiating minor compaction (all files) 2024-11-20T09:01:46,087 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e765ef86c58446bc8e368358023e83a1/B in TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1. 2024-11-20T09:01:46,087 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e765ef86c58446bc8e368358023e83a1/A in TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1. 2024-11-20T09:01:46,087 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/B/cd613d42ffda4d22994af7647f69ae48, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/B/831a3e5cede142d4b7067895861d17b0, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/B/d57d1a7799904fb8a183bb0530282a07] into tmpdir=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp, totalSize=36.9 K 2024-11-20T09:01:46,087 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/A/70ca48fde567446cae66819396c5cd18, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/A/493a3247619c4385a5b8b402d784bf60, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/A/8dfd75f108464e2ba593fd8fbd6e6625] into tmpdir=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp, totalSize=36.9 K 2024-11-20T09:01:46,088 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting cd613d42ffda4d22994af7647f69ae48, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=349, earliestPutTs=1732093300769 2024-11-20T09:01:46,088 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.Compactor(225): Compacting 70ca48fde567446cae66819396c5cd18, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=349, earliestPutTs=1732093300769 2024-11-20T09:01:46,088 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting 831a3e5cede142d4b7067895861d17b0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=366, earliestPutTs=1732093301974 2024-11-20T09:01:46,088 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.Compactor(225): Compacting 493a3247619c4385a5b8b402d784bf60, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=366, earliestPutTs=1732093301974 2024-11-20T09:01:46,088 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting d57d1a7799904fb8a183bb0530282a07, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=390, earliestPutTs=1732093303692 2024-11-20T09:01:46,088 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8dfd75f108464e2ba593fd8fbd6e6625, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=390, earliestPutTs=1732093303692 2024-11-20T09:01:46,096 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e765ef86c58446bc8e368358023e83a1#A#compaction#474 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T09:01:46,096 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e765ef86c58446bc8e368358023e83a1#B#compaction#473 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T09:01:46,096 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/A/022d78ed79dd42ca9eb7fb0e9893df18 is 50, key is test_row_0/A:col10/1732093303692/Put/seqid=0 2024-11-20T09:01:46,096 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/B/413b7273131340168b93ec5adc309410 is 50, key is test_row_0/B:col10/1732093303692/Put/seqid=0 2024-11-20T09:01:46,099 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742375_1551 (size=13255) 2024-11-20T09:01:46,100 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742376_1552 (size=13255) 2024-11-20T09:01:46,178 INFO [Thread-2091 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2091. 2024-11-20T09:01:46,178 DEBUG [Thread-2091 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T09:01:46,178 DEBUG [Thread-2091 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T09:01:46,178 DEBUG [Thread-2091 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T09:01:46,178 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T09:01:46,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-20T09:01:46,217 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T09:01:46,218 INFO [Thread-2089 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2089. 2024-11-20T09:01:46,218 DEBUG [Thread-2089 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T09:01:46,218 DEBUG [Thread-2089 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T09:01:46,218 DEBUG [Thread-2089 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T09:01:46,218 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T09:01:46,503 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/A/022d78ed79dd42ca9eb7fb0e9893df18 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/A/022d78ed79dd42ca9eb7fb0e9893df18 2024-11-20T09:01:46,503 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/B/413b7273131340168b93ec5adc309410 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/B/413b7273131340168b93ec5adc309410 2024-11-20T09:01:46,507 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e765ef86c58446bc8e368358023e83a1/A of e765ef86c58446bc8e368358023e83a1 into 022d78ed79dd42ca9eb7fb0e9893df18(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T09:01:46,507 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e765ef86c58446bc8e368358023e83a1/B of e765ef86c58446bc8e368358023e83a1 into 413b7273131340168b93ec5adc309410(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T09:01:46,507 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e765ef86c58446bc8e368358023e83a1: 2024-11-20T09:01:46,507 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e765ef86c58446bc8e368358023e83a1: 2024-11-20T09:01:46,507 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., storeName=e765ef86c58446bc8e368358023e83a1/A, priority=13, startTime=1732093306086; duration=0sec 2024-11-20T09:01:46,507 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., storeName=e765ef86c58446bc8e368358023e83a1/B, priority=13, startTime=1732093306086; duration=0sec 2024-11-20T09:01:46,507 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T09:01:46,507 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T09:01:46,507 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e765ef86c58446bc8e368358023e83a1:B 2024-11-20T09:01:46,507 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e765ef86c58446bc8e368358023e83a1:A 2024-11-20T09:01:46,507 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T09:01:46,508 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T09:01:46,508 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1541): e765ef86c58446bc8e368358023e83a1/C is initiating minor compaction (all files) 2024-11-20T09:01:46,508 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e765ef86c58446bc8e368358023e83a1/C in TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1. 2024-11-20T09:01:46,508 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/C/977dffe95a934a9c80e9a8f7a2779128, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/C/2a1459a8998445038f33255c514a5fd2, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/C/f007d72587354260803eae9ff3d2caee] into tmpdir=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp, totalSize=36.9 K 2024-11-20T09:01:46,508 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting 977dffe95a934a9c80e9a8f7a2779128, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=349, earliestPutTs=1732093300769 2024-11-20T09:01:46,508 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting 2a1459a8998445038f33255c514a5fd2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=366, earliestPutTs=1732093301974 2024-11-20T09:01:46,508 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting f007d72587354260803eae9ff3d2caee, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=390, earliestPutTs=1732093303692 2024-11-20T09:01:46,514 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e765ef86c58446bc8e368358023e83a1#C#compaction#475 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T09:01:46,514 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/C/7e12e1164410419280e1580fd2c3d718 is 50, key is test_row_0/C:col10/1732093303692/Put/seqid=0 2024-11-20T09:01:46,517 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742377_1553 (size=13255) 2024-11-20T09:01:46,538 INFO [Thread-2087 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2087. 2024-11-20T09:01:46,538 DEBUG [Thread-2087 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T09:01:46,538 DEBUG [Thread-2087 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T09:01:46,538 DEBUG [Thread-2087 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T09:01:46,538 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T09:01:46,538 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-20T09:01:46,538 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 67 2024-11-20T09:01:46,538 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 60 2024-11-20T09:01:46,538 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 23 2024-11-20T09:01:46,538 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 87 2024-11-20T09:01:46,538 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 71 2024-11-20T09:01:46,538 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-20T09:01:46,538 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-20T09:01:46,538 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1162 2024-11-20T09:01:46,538 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3486 rows 2024-11-20T09:01:46,538 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1154 2024-11-20T09:01:46,538 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3459 rows 2024-11-20T09:01:46,538 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1155 2024-11-20T09:01:46,538 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3465 rows 2024-11-20T09:01:46,538 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1154 2024-11-20T09:01:46,538 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3461 rows 2024-11-20T09:01:46,538 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1152 2024-11-20T09:01:46,538 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3456 rows 2024-11-20T09:01:46,538 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-20T09:01:46,538 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testScanAtomicity(AcidGuaranteesTestBase.java:112) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-20T09:01:46,538 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T09:01:46,538 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T09:01:46,539 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T09:01:46,539 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-20T09:01:46,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] procedure2.ProcedureExecutor(1139): Stored pid=131, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-20T09:01:46,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-20T09:01:46,542 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732093306542"}]},"ts":"1732093306542"} 2024-11-20T09:01:46,543 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-20T09:01:46,543 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-20T09:01:46,544 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=132, ppid=131, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-20T09:01:46,545 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=133, ppid=132, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e765ef86c58446bc8e368358023e83a1, UNASSIGN}] 2024-11-20T09:01:46,545 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=133, ppid=132, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e765ef86c58446bc8e368358023e83a1, UNASSIGN 2024-11-20T09:01:46,546 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=133 updating hbase:meta row=e765ef86c58446bc8e368358023e83a1, regionState=CLOSING, regionLocation=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:46,547 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=133, ppid=132, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e765ef86c58446bc8e368358023e83a1, UNASSIGN because future has completed 2024-11-20T09:01:46,547 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-20T09:01:46,547 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=134, ppid=133, state=RUNNABLE, hasLock=false; CloseRegionProcedure e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960}] 2024-11-20T09:01:46,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-20T09:01:46,699 INFO [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] handler.UnassignRegionHandler(122): Close e765ef86c58446bc8e368358023e83a1 2024-11-20T09:01:46,699 DEBUG [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-20T09:01:46,699 DEBUG [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegion(1722): Closing e765ef86c58446bc8e368358023e83a1, disabling compactions & flushes 2024-11-20T09:01:46,699 DEBUG [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegion(1993): waiting for 1 compactions to complete for region TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1. 2024-11-20T09:01:46,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-20T09:01:46,921 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/C/7e12e1164410419280e1580fd2c3d718 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/C/7e12e1164410419280e1580fd2c3d718 2024-11-20T09:01:46,924 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e765ef86c58446bc8e368358023e83a1/C of e765ef86c58446bc8e368358023e83a1 into 7e12e1164410419280e1580fd2c3d718(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T09:01:46,924 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e765ef86c58446bc8e368358023e83a1: 2024-11-20T09:01:46,924 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., storeName=e765ef86c58446bc8e368358023e83a1/C, priority=13, startTime=1732093306086; duration=0sec 2024-11-20T09:01:46,924 INFO [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1. 2024-11-20T09:01:46,924 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T09:01:46,924 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e765ef86c58446bc8e368358023e83a1:C 2024-11-20T09:01:46,925 DEBUG [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1. 2024-11-20T09:01:46,925 DEBUG [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1. after waiting 0 ms 2024-11-20T09:01:46,925 DEBUG [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1. 2024-11-20T09:01:46,925 INFO [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegion(2902): Flushing e765ef86c58446bc8e368358023e83a1 3/3 column families, dataSize=20.13 KB heapSize=53.48 KB 2024-11-20T09:01:46,928 DEBUG [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/A/94ffa029ed884e439ab7a895947bdd5d is 50, key is test_row_0/A:col10/1732093306537/Put/seqid=0 2024-11-20T09:01:46,930 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742378_1554 (size=9857) 2024-11-20T09:01:47,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-20T09:01:47,331 INFO [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=399 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/A/94ffa029ed884e439ab7a895947bdd5d 2024-11-20T09:01:47,336 DEBUG [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/B/dad8c7481d5942f2b8dee3d17c51e257 is 50, key is test_row_0/B:col10/1732093306537/Put/seqid=0 2024-11-20T09:01:47,339 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742379_1555 (size=9857) 2024-11-20T09:01:47,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-20T09:01:47,740 INFO [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=399 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/B/dad8c7481d5942f2b8dee3d17c51e257 2024-11-20T09:01:47,745 DEBUG [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/C/d2eb166bda854a38867a83f021f9d556 is 50, key is test_row_0/C:col10/1732093306537/Put/seqid=0 2024-11-20T09:01:47,747 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742380_1556 (size=9857) 2024-11-20T09:01:48,148 INFO [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=399 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/C/d2eb166bda854a38867a83f021f9d556 2024-11-20T09:01:48,151 DEBUG [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/A/94ffa029ed884e439ab7a895947bdd5d as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/A/94ffa029ed884e439ab7a895947bdd5d 2024-11-20T09:01:48,154 INFO [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/A/94ffa029ed884e439ab7a895947bdd5d, entries=100, sequenceid=399, filesize=9.6 K 2024-11-20T09:01:48,155 DEBUG [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/B/dad8c7481d5942f2b8dee3d17c51e257 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/B/dad8c7481d5942f2b8dee3d17c51e257 2024-11-20T09:01:48,157 INFO [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/B/dad8c7481d5942f2b8dee3d17c51e257, entries=100, sequenceid=399, filesize=9.6 K 2024-11-20T09:01:48,158 DEBUG [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/.tmp/C/d2eb166bda854a38867a83f021f9d556 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/C/d2eb166bda854a38867a83f021f9d556 2024-11-20T09:01:48,160 INFO [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/C/d2eb166bda854a38867a83f021f9d556, entries=100, sequenceid=399, filesize=9.6 K 2024-11-20T09:01:48,161 INFO [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~53.44 KB/54720, currentSize=0 B/0 for e765ef86c58446bc8e368358023e83a1 in 1236ms, sequenceid=399, compaction requested=false 2024-11-20T09:01:48,161 DEBUG [StoreCloser-TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/A/4bcfa11498164d259adc42674aa877aa, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/A/e8defb143a7f4be3be7b3cff158d70c4, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/A/3ffc2f0d13a343048bbbf9e77d1d95ce, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/A/77908ad062e74187b8bf9466aef85d48, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/A/6de8a38311a44bc88291f16ca0a48e3b, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/A/44449f6c51fb4e0e9f63c4358bede375, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/A/56fb634a8cd646e0987e34ed4d457fce, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/A/7930cf28cb00469591a56b01b26e5f3a, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/A/1410d9495a084237a4dfc31c3be0b624, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/A/6568c6826c394fc98f3818a5a8d672ea, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/A/7b7c5943defa49b29deff103c5ad984c, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/A/40ecb445f26e41ec898344699923b391, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/A/6cd49cde118347c39d392cea52f2643f, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/A/ee8140cc1c4849c6824b05bdf362781b, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/A/7d28d4f828424e8ab750783d8808a5b9, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/A/3ded0b6642764516989f24b70464f481, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/A/eb633c56abaa480ca9dc42ee7c2757be, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/A/4738cbe7563a442f859e5a0216e1b04f, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/A/f2f72fc1bd5945ea81c3160a0b220707, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/A/e52ac2ecf4bd4820823c83c6b97a4e3e, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/A/16c959a1e6b744608527ad48d5c9f7df, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/A/5b93cf58f8594e35914ec9c4bba4637f, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/A/590d4880903248979585284bf2dab80e, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/A/72d7dac81ace448095c1e9ab3949b16d, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/A/21a5d27bb00444e2be3dd3439cd1095b, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/A/70ca48fde567446cae66819396c5cd18, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/A/493a3247619c4385a5b8b402d784bf60, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/A/8dfd75f108464e2ba593fd8fbd6e6625] to archive 2024-11-20T09:01:48,162 DEBUG [StoreCloser-TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-20T09:01:48,163 DEBUG [StoreCloser-TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/A/4bcfa11498164d259adc42674aa877aa to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/A/4bcfa11498164d259adc42674aa877aa 2024-11-20T09:01:48,164 DEBUG [StoreCloser-TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/A/e8defb143a7f4be3be7b3cff158d70c4 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/A/e8defb143a7f4be3be7b3cff158d70c4 2024-11-20T09:01:48,165 DEBUG [StoreCloser-TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/A/3ffc2f0d13a343048bbbf9e77d1d95ce to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/A/3ffc2f0d13a343048bbbf9e77d1d95ce 2024-11-20T09:01:48,166 DEBUG [StoreCloser-TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/A/77908ad062e74187b8bf9466aef85d48 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/A/77908ad062e74187b8bf9466aef85d48 2024-11-20T09:01:48,166 DEBUG [StoreCloser-TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/A/6de8a38311a44bc88291f16ca0a48e3b to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/A/6de8a38311a44bc88291f16ca0a48e3b 2024-11-20T09:01:48,167 DEBUG [StoreCloser-TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/A/44449f6c51fb4e0e9f63c4358bede375 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/A/44449f6c51fb4e0e9f63c4358bede375 2024-11-20T09:01:48,168 DEBUG [StoreCloser-TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/A/56fb634a8cd646e0987e34ed4d457fce to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/A/56fb634a8cd646e0987e34ed4d457fce 2024-11-20T09:01:48,169 DEBUG [StoreCloser-TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/A/7930cf28cb00469591a56b01b26e5f3a to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/A/7930cf28cb00469591a56b01b26e5f3a 2024-11-20T09:01:48,169 DEBUG [StoreCloser-TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/A/1410d9495a084237a4dfc31c3be0b624 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/A/1410d9495a084237a4dfc31c3be0b624 2024-11-20T09:01:48,170 DEBUG [StoreCloser-TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/A/6568c6826c394fc98f3818a5a8d672ea to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/A/6568c6826c394fc98f3818a5a8d672ea 2024-11-20T09:01:48,171 DEBUG [StoreCloser-TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/A/7b7c5943defa49b29deff103c5ad984c to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/A/7b7c5943defa49b29deff103c5ad984c 2024-11-20T09:01:48,172 DEBUG [StoreCloser-TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/A/40ecb445f26e41ec898344699923b391 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/A/40ecb445f26e41ec898344699923b391 2024-11-20T09:01:48,172 DEBUG [StoreCloser-TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/A/6cd49cde118347c39d392cea52f2643f to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/A/6cd49cde118347c39d392cea52f2643f 2024-11-20T09:01:48,173 DEBUG [StoreCloser-TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/A/ee8140cc1c4849c6824b05bdf362781b to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/A/ee8140cc1c4849c6824b05bdf362781b 2024-11-20T09:01:48,174 DEBUG [StoreCloser-TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/A/7d28d4f828424e8ab750783d8808a5b9 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/A/7d28d4f828424e8ab750783d8808a5b9 2024-11-20T09:01:48,175 DEBUG [StoreCloser-TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/A/3ded0b6642764516989f24b70464f481 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/A/3ded0b6642764516989f24b70464f481 2024-11-20T09:01:48,175 DEBUG [StoreCloser-TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/A/eb633c56abaa480ca9dc42ee7c2757be to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/A/eb633c56abaa480ca9dc42ee7c2757be 2024-11-20T09:01:48,176 DEBUG [StoreCloser-TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/A/4738cbe7563a442f859e5a0216e1b04f to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/A/4738cbe7563a442f859e5a0216e1b04f 2024-11-20T09:01:48,177 DEBUG [StoreCloser-TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/A/f2f72fc1bd5945ea81c3160a0b220707 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/A/f2f72fc1bd5945ea81c3160a0b220707 2024-11-20T09:01:48,178 DEBUG [StoreCloser-TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/A/e52ac2ecf4bd4820823c83c6b97a4e3e to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/A/e52ac2ecf4bd4820823c83c6b97a4e3e 2024-11-20T09:01:48,178 DEBUG [StoreCloser-TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/A/16c959a1e6b744608527ad48d5c9f7df to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/A/16c959a1e6b744608527ad48d5c9f7df 2024-11-20T09:01:48,179 DEBUG [StoreCloser-TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/A/5b93cf58f8594e35914ec9c4bba4637f to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/A/5b93cf58f8594e35914ec9c4bba4637f 2024-11-20T09:01:48,180 DEBUG [StoreCloser-TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/A/590d4880903248979585284bf2dab80e to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/A/590d4880903248979585284bf2dab80e 2024-11-20T09:01:48,181 DEBUG [StoreCloser-TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/A/72d7dac81ace448095c1e9ab3949b16d to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/A/72d7dac81ace448095c1e9ab3949b16d 2024-11-20T09:01:48,182 DEBUG [StoreCloser-TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/A/21a5d27bb00444e2be3dd3439cd1095b to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/A/21a5d27bb00444e2be3dd3439cd1095b 2024-11-20T09:01:48,182 DEBUG [StoreCloser-TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/A/70ca48fde567446cae66819396c5cd18 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/A/70ca48fde567446cae66819396c5cd18 2024-11-20T09:01:48,183 DEBUG [StoreCloser-TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/A/493a3247619c4385a5b8b402d784bf60 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/A/493a3247619c4385a5b8b402d784bf60 2024-11-20T09:01:48,184 DEBUG [StoreCloser-TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/A/8dfd75f108464e2ba593fd8fbd6e6625 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/A/8dfd75f108464e2ba593fd8fbd6e6625 2024-11-20T09:01:48,185 DEBUG [StoreCloser-TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/B/66e55a5c10844f85926e7466cc478c35, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/B/64af32b0fdd3480fb334d6c518434b67, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/B/697b5306184b4b1e8541e75ab8f7670a, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/B/79c5fe4a80fc4d31bc261f8494f38eae, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/B/6e49b35edd094ed880e578eb9ed9a94d, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/B/dcb8ea1cb8f34f3892bde30386b04ef2, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/B/d908eb4390cd4940ba7b5fe463c91ba1, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/B/9d9902e4ddea40a4888ab266b0103afb, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/B/783e6f881d73425397869ab97184c4d3, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/B/5276e1e67c764a38a60d55e07396cdb4, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/B/668bfdebcb7b46b1bb24009a1309b41f, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/B/08e93114d7d34d6abfe76efda84d5177, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/B/e117db64b6c14c708fc2acf2b513a3d9, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/B/bcc8a034088348aba915b67edd0f4567, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/B/ea3c53c0b9d94ef18438e032036f69b6, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/B/d3a78d5597704ad981fad51330cc19e3, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/B/1be6026e6cd7422bb1f4c335a8d4dcb5, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/B/cf9f2ce06d4842a68c7b3dcd66beb210, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/B/fce689c164c34bed937078efb069e61c, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/B/489cff995f8f4800b30c244b30bbadb2, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/B/642e16184be0436ea1fed4ca81af525e, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/B/b4ff984fcc284d62bf4c4330d5998b89, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/B/9f01e0eaf92b4b8cb73ad682d7a024b2, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/B/e661acf1781c4c75863417a68efeb780, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/B/cd613d42ffda4d22994af7647f69ae48, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/B/68518345671e41218a48b5495b49e296, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/B/831a3e5cede142d4b7067895861d17b0, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/B/d57d1a7799904fb8a183bb0530282a07] to archive 2024-11-20T09:01:48,186 DEBUG [StoreCloser-TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-20T09:01:48,187 DEBUG [StoreCloser-TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/B/66e55a5c10844f85926e7466cc478c35 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/B/66e55a5c10844f85926e7466cc478c35 2024-11-20T09:01:48,188 DEBUG [StoreCloser-TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/B/64af32b0fdd3480fb334d6c518434b67 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/B/64af32b0fdd3480fb334d6c518434b67 2024-11-20T09:01:48,188 DEBUG [StoreCloser-TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/B/697b5306184b4b1e8541e75ab8f7670a to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/B/697b5306184b4b1e8541e75ab8f7670a 2024-11-20T09:01:48,189 DEBUG [StoreCloser-TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/B/79c5fe4a80fc4d31bc261f8494f38eae to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/B/79c5fe4a80fc4d31bc261f8494f38eae 2024-11-20T09:01:48,190 DEBUG [StoreCloser-TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/B/6e49b35edd094ed880e578eb9ed9a94d to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/B/6e49b35edd094ed880e578eb9ed9a94d 2024-11-20T09:01:48,191 DEBUG [StoreCloser-TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/B/dcb8ea1cb8f34f3892bde30386b04ef2 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/B/dcb8ea1cb8f34f3892bde30386b04ef2 2024-11-20T09:01:48,191 DEBUG [StoreCloser-TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/B/d908eb4390cd4940ba7b5fe463c91ba1 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/B/d908eb4390cd4940ba7b5fe463c91ba1 2024-11-20T09:01:48,192 DEBUG [StoreCloser-TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/B/9d9902e4ddea40a4888ab266b0103afb to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/B/9d9902e4ddea40a4888ab266b0103afb 2024-11-20T09:01:48,193 DEBUG [StoreCloser-TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/B/783e6f881d73425397869ab97184c4d3 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/B/783e6f881d73425397869ab97184c4d3 2024-11-20T09:01:48,193 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-20T09:01:48,194 DEBUG [StoreCloser-TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/B/5276e1e67c764a38a60d55e07396cdb4 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/B/5276e1e67c764a38a60d55e07396cdb4 2024-11-20T09:01:48,194 DEBUG [StoreCloser-TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/B/668bfdebcb7b46b1bb24009a1309b41f to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/B/668bfdebcb7b46b1bb24009a1309b41f 2024-11-20T09:01:48,195 DEBUG [StoreCloser-TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/B/08e93114d7d34d6abfe76efda84d5177 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/B/08e93114d7d34d6abfe76efda84d5177 2024-11-20T09:01:48,196 DEBUG [StoreCloser-TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/B/e117db64b6c14c708fc2acf2b513a3d9 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/B/e117db64b6c14c708fc2acf2b513a3d9 2024-11-20T09:01:48,197 DEBUG [StoreCloser-TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/B/bcc8a034088348aba915b67edd0f4567 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/B/bcc8a034088348aba915b67edd0f4567 2024-11-20T09:01:48,198 DEBUG [StoreCloser-TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/B/ea3c53c0b9d94ef18438e032036f69b6 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/B/ea3c53c0b9d94ef18438e032036f69b6 2024-11-20T09:01:48,198 DEBUG [StoreCloser-TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/B/d3a78d5597704ad981fad51330cc19e3 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/B/d3a78d5597704ad981fad51330cc19e3 2024-11-20T09:01:48,199 DEBUG [StoreCloser-TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/B/1be6026e6cd7422bb1f4c335a8d4dcb5 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/B/1be6026e6cd7422bb1f4c335a8d4dcb5 2024-11-20T09:01:48,200 DEBUG [StoreCloser-TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/B/cf9f2ce06d4842a68c7b3dcd66beb210 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/B/cf9f2ce06d4842a68c7b3dcd66beb210 2024-11-20T09:01:48,201 DEBUG [StoreCloser-TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/B/fce689c164c34bed937078efb069e61c to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/B/fce689c164c34bed937078efb069e61c 2024-11-20T09:01:48,202 DEBUG [StoreCloser-TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/B/489cff995f8f4800b30c244b30bbadb2 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/B/489cff995f8f4800b30c244b30bbadb2 2024-11-20T09:01:48,203 DEBUG [StoreCloser-TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/B/642e16184be0436ea1fed4ca81af525e to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/B/642e16184be0436ea1fed4ca81af525e 2024-11-20T09:01:48,203 DEBUG [StoreCloser-TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/B/b4ff984fcc284d62bf4c4330d5998b89 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/B/b4ff984fcc284d62bf4c4330d5998b89 2024-11-20T09:01:48,204 DEBUG [StoreCloser-TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/B/9f01e0eaf92b4b8cb73ad682d7a024b2 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/B/9f01e0eaf92b4b8cb73ad682d7a024b2 2024-11-20T09:01:48,205 DEBUG [StoreCloser-TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/B/e661acf1781c4c75863417a68efeb780 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/B/e661acf1781c4c75863417a68efeb780 2024-11-20T09:01:48,206 DEBUG [StoreCloser-TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/B/cd613d42ffda4d22994af7647f69ae48 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/B/cd613d42ffda4d22994af7647f69ae48 2024-11-20T09:01:48,207 DEBUG [StoreCloser-TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/B/68518345671e41218a48b5495b49e296 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/B/68518345671e41218a48b5495b49e296 2024-11-20T09:01:48,208 DEBUG [StoreCloser-TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/B/831a3e5cede142d4b7067895861d17b0 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/B/831a3e5cede142d4b7067895861d17b0 2024-11-20T09:01:48,209 DEBUG [StoreCloser-TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/B/d57d1a7799904fb8a183bb0530282a07 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/B/d57d1a7799904fb8a183bb0530282a07 2024-11-20T09:01:48,210 DEBUG [StoreCloser-TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/C/cb5737d0e45c4fd291bfaa472ea1094f, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/C/a1de1814bc8d4ca7bca9174a70789e78, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/C/a3d7531c88fd4c1088abd241acd76c64, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/C/bf8037924a7a4c718b05880d914f422e, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/C/e646e522f7b34fc298e8c4571d11232d, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/C/b606d8d0977e41758458c67f49385797, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/C/eb53dbb857d941fc97c2e076e36b67ca, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/C/f9860385e33d4af6b0840ef0c0650bc8, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/C/6ce0f266b563427888ede01e8c78ffe4, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/C/ae362a02de4546d19d7c7b5d0bfa42a4, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/C/b1934a272e314560b9466012b62afde3, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/C/eb23e2df7043431eb796f695d2912a64, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/C/5087a54e2d79408c9fba20f7207cc7e5, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/C/8c6e4efb5b5a44b79deb05f1dd364f6a, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/C/3e86b9bd75fa4391a1cab6fef518dd8b, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/C/afbb338dbfbc40f1b3be7078dcc0d9ee, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/C/174ce5669bac443c8a3a58c446e4e80c, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/C/a43a3b186a5941b6b71fb3a4475a18fe, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/C/aabf66da690a4c57a09c79d93bc18a90, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/C/52bc9be5a58d4423800e6269c8c83487, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/C/c627a7cf687e425cad6d964ed0272c35, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/C/b6096eea23224fd693fb846b7bb820d2, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/C/d1aba9fb1c974dc9a10390119e94efde, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/C/6f48b3fabfe44e90aa01b6e8213502dd, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/C/977dffe95a934a9c80e9a8f7a2779128, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/C/f2f062464cc64f98b2a617e7ad21599c, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/C/2a1459a8998445038f33255c514a5fd2, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/C/f007d72587354260803eae9ff3d2caee] to archive 2024-11-20T09:01:48,211 DEBUG [StoreCloser-TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-20T09:01:48,212 DEBUG [StoreCloser-TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/C/cb5737d0e45c4fd291bfaa472ea1094f to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/C/cb5737d0e45c4fd291bfaa472ea1094f 2024-11-20T09:01:48,213 DEBUG [StoreCloser-TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/C/a1de1814bc8d4ca7bca9174a70789e78 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/C/a1de1814bc8d4ca7bca9174a70789e78 2024-11-20T09:01:48,214 DEBUG [StoreCloser-TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/C/a3d7531c88fd4c1088abd241acd76c64 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/C/a3d7531c88fd4c1088abd241acd76c64 2024-11-20T09:01:48,214 DEBUG [StoreCloser-TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/C/bf8037924a7a4c718b05880d914f422e to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/C/bf8037924a7a4c718b05880d914f422e 2024-11-20T09:01:48,215 DEBUG [StoreCloser-TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/C/e646e522f7b34fc298e8c4571d11232d to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/C/e646e522f7b34fc298e8c4571d11232d 2024-11-20T09:01:48,216 DEBUG [StoreCloser-TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/C/b606d8d0977e41758458c67f49385797 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/C/b606d8d0977e41758458c67f49385797 2024-11-20T09:01:48,217 DEBUG [StoreCloser-TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/C/eb53dbb857d941fc97c2e076e36b67ca to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/C/eb53dbb857d941fc97c2e076e36b67ca 2024-11-20T09:01:48,218 DEBUG [StoreCloser-TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/C/f9860385e33d4af6b0840ef0c0650bc8 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/C/f9860385e33d4af6b0840ef0c0650bc8 2024-11-20T09:01:48,218 DEBUG [StoreCloser-TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/C/6ce0f266b563427888ede01e8c78ffe4 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/C/6ce0f266b563427888ede01e8c78ffe4 2024-11-20T09:01:48,219 DEBUG [StoreCloser-TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/C/ae362a02de4546d19d7c7b5d0bfa42a4 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/C/ae362a02de4546d19d7c7b5d0bfa42a4 2024-11-20T09:01:48,220 DEBUG [StoreCloser-TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/C/b1934a272e314560b9466012b62afde3 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/C/b1934a272e314560b9466012b62afde3 2024-11-20T09:01:48,221 DEBUG [StoreCloser-TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/C/eb23e2df7043431eb796f695d2912a64 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/C/eb23e2df7043431eb796f695d2912a64 2024-11-20T09:01:48,222 DEBUG [StoreCloser-TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/C/5087a54e2d79408c9fba20f7207cc7e5 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/C/5087a54e2d79408c9fba20f7207cc7e5 2024-11-20T09:01:48,222 DEBUG [StoreCloser-TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/C/8c6e4efb5b5a44b79deb05f1dd364f6a to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/C/8c6e4efb5b5a44b79deb05f1dd364f6a 2024-11-20T09:01:48,223 DEBUG [StoreCloser-TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/C/3e86b9bd75fa4391a1cab6fef518dd8b to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/C/3e86b9bd75fa4391a1cab6fef518dd8b 2024-11-20T09:01:48,224 DEBUG [StoreCloser-TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/C/afbb338dbfbc40f1b3be7078dcc0d9ee to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/C/afbb338dbfbc40f1b3be7078dcc0d9ee 2024-11-20T09:01:48,225 DEBUG [StoreCloser-TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/C/174ce5669bac443c8a3a58c446e4e80c to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/C/174ce5669bac443c8a3a58c446e4e80c 2024-11-20T09:01:48,226 DEBUG [StoreCloser-TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/C/a43a3b186a5941b6b71fb3a4475a18fe to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/C/a43a3b186a5941b6b71fb3a4475a18fe 2024-11-20T09:01:48,226 DEBUG [StoreCloser-TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/C/aabf66da690a4c57a09c79d93bc18a90 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/C/aabf66da690a4c57a09c79d93bc18a90 2024-11-20T09:01:48,227 DEBUG [StoreCloser-TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/C/52bc9be5a58d4423800e6269c8c83487 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/C/52bc9be5a58d4423800e6269c8c83487 2024-11-20T09:01:48,227 DEBUG [StoreCloser-TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/C/c627a7cf687e425cad6d964ed0272c35 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/C/c627a7cf687e425cad6d964ed0272c35 2024-11-20T09:01:48,228 DEBUG [StoreCloser-TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/C/b6096eea23224fd693fb846b7bb820d2 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/C/b6096eea23224fd693fb846b7bb820d2 2024-11-20T09:01:48,229 DEBUG [StoreCloser-TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/C/d1aba9fb1c974dc9a10390119e94efde to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/C/d1aba9fb1c974dc9a10390119e94efde 2024-11-20T09:01:48,230 DEBUG [StoreCloser-TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/C/6f48b3fabfe44e90aa01b6e8213502dd to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/C/6f48b3fabfe44e90aa01b6e8213502dd 2024-11-20T09:01:48,231 DEBUG [StoreCloser-TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/C/977dffe95a934a9c80e9a8f7a2779128 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/C/977dffe95a934a9c80e9a8f7a2779128 2024-11-20T09:01:48,232 DEBUG [StoreCloser-TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/C/f2f062464cc64f98b2a617e7ad21599c to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/C/f2f062464cc64f98b2a617e7ad21599c 2024-11-20T09:01:48,233 DEBUG [StoreCloser-TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/C/2a1459a8998445038f33255c514a5fd2 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/C/2a1459a8998445038f33255c514a5fd2 2024-11-20T09:01:48,234 DEBUG [StoreCloser-TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/C/f007d72587354260803eae9ff3d2caee to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/C/f007d72587354260803eae9ff3d2caee 2024-11-20T09:01:48,237 DEBUG [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/recovered.edits/402.seqid, newMaxSeqId=402, maxSeqId=1 2024-11-20T09:01:48,238 INFO [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1. 2024-11-20T09:01:48,238 DEBUG [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegion(1676): Region close journal for e765ef86c58446bc8e368358023e83a1: Waiting for close lock at 1732093306699Running coprocessor pre-close hooks at 1732093306699Disabling compacts and flushes for region at 1732093306699Disabling writes for close at 1732093306925 (+226 ms)Obtaining lock to block concurrent updates at 1732093306925Preparing flush snapshotting stores in e765ef86c58446bc8e368358023e83a1 at 1732093306925Finished memstore snapshotting TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1., syncing WAL and waiting on mvcc, flushsize=dataSize=20610, getHeapSize=54720, getOffHeapSize=0, getCellsCount=450 at 1732093306925Flushing stores of TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1. at 1732093306925Flushing e765ef86c58446bc8e368358023e83a1/A: creating writer at 1732093306925Flushing e765ef86c58446bc8e368358023e83a1/A: appending metadata at 1732093306927 (+2 ms)Flushing e765ef86c58446bc8e368358023e83a1/A: closing flushed file at 1732093306927Flushing e765ef86c58446bc8e368358023e83a1/B: creating writer at 1732093307334 (+407 ms)Flushing e765ef86c58446bc8e368358023e83a1/B: appending metadata at 1732093307336 (+2 ms)Flushing e765ef86c58446bc8e368358023e83a1/B: closing flushed file at 1732093307336Flushing e765ef86c58446bc8e368358023e83a1/C: creating writer at 1732093307742 (+406 ms)Flushing e765ef86c58446bc8e368358023e83a1/C: appending metadata at 1732093307744 (+2 ms)Flushing e765ef86c58446bc8e368358023e83a1/C: closing flushed file at 1732093307744Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@338622b3: reopening flushed file at 1732093308151 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@146c5e85: reopening flushed file at 1732093308154 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4f6d233e: reopening flushed file at 1732093308157 (+3 ms)Finished flush of dataSize ~20.13 KB/20610, heapSize ~53.44 KB/54720, currentSize=0 B/0 for e765ef86c58446bc8e368358023e83a1 in 1236ms, sequenceid=399, compaction requested=false at 1732093308161 (+4 ms)Writing region close event to WAL at 1732093308235 (+74 ms)Running coprocessor post-close hooks at 1732093308238 (+3 ms)Closed at 1732093308238 2024-11-20T09:01:48,240 INFO [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] handler.UnassignRegionHandler(157): Closed e765ef86c58446bc8e368358023e83a1 2024-11-20T09:01:48,240 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=133 updating hbase:meta row=e765ef86c58446bc8e368358023e83a1, regionState=CLOSED 2024-11-20T09:01:48,242 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=134, ppid=133, state=RUNNABLE, hasLock=false; CloseRegionProcedure e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 because future has completed 2024-11-20T09:01:48,243 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=134, resume processing ppid=133 2024-11-20T09:01:48,243 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=134, ppid=133, state=SUCCESS, hasLock=false; CloseRegionProcedure e765ef86c58446bc8e368358023e83a1, server=be1eb2620e0e,34223,1732093159960 in 1.6950 sec 2024-11-20T09:01:48,245 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=133, resume processing ppid=132 2024-11-20T09:01:48,245 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=133, ppid=132, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e765ef86c58446bc8e368358023e83a1, UNASSIGN in 1.6980 sec 2024-11-20T09:01:48,247 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=132, resume processing ppid=131 2024-11-20T09:01:48,247 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=132, ppid=131, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.7020 sec 2024-11-20T09:01:48,248 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732093308248"}]},"ts":"1732093308248"} 2024-11-20T09:01:48,249 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-20T09:01:48,249 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-20T09:01:48,250 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=131, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.7100 sec 2024-11-20T09:01:48,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-20T09:01:48,687 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-20T09:01:48,688 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-20T09:01:48,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] procedure2.ProcedureExecutor(1139): Stored pid=135, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T09:01:48,689 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=135, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T09:01:48,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-20T09:01:48,690 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=135, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T09:01:48,691 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1 2024-11-20T09:01:48,693 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/A, FileablePath, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/B, FileablePath, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/C, FileablePath, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/recovered.edits] 2024-11-20T09:01:48,695 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/A/022d78ed79dd42ca9eb7fb0e9893df18 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/A/022d78ed79dd42ca9eb7fb0e9893df18 2024-11-20T09:01:48,696 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/A/94ffa029ed884e439ab7a895947bdd5d to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/A/94ffa029ed884e439ab7a895947bdd5d 2024-11-20T09:01:48,697 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/B/413b7273131340168b93ec5adc309410 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/B/413b7273131340168b93ec5adc309410 2024-11-20T09:01:48,698 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/B/dad8c7481d5942f2b8dee3d17c51e257 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/B/dad8c7481d5942f2b8dee3d17c51e257 2024-11-20T09:01:48,700 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/C/7e12e1164410419280e1580fd2c3d718 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/C/7e12e1164410419280e1580fd2c3d718 2024-11-20T09:01:48,701 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/C/d2eb166bda854a38867a83f021f9d556 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/C/d2eb166bda854a38867a83f021f9d556 2024-11-20T09:01:48,702 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/recovered.edits/402.seqid to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1/recovered.edits/402.seqid 2024-11-20T09:01:48,703 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/e765ef86c58446bc8e368358023e83a1 2024-11-20T09:01:48,703 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-20T09:01:48,705 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=135, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T09:01:48,706 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-20T09:01:48,708 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-20T09:01:48,708 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=135, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T09:01:48,708 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-20T09:01:48,709 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732093308708"}]},"ts":"9223372036854775807"} 2024-11-20T09:01:48,710 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-20T09:01:48,710 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => e765ef86c58446bc8e368358023e83a1, NAME => 'TestAcidGuarantees,,1732093281799.e765ef86c58446bc8e368358023e83a1.', STARTKEY => '', ENDKEY => ''}] 2024-11-20T09:01:48,710 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-20T09:01:48,710 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732093308710"}]},"ts":"9223372036854775807"} 2024-11-20T09:01:48,712 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-20T09:01:48,712 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=135, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T09:01:48,713 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=135, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 24 msec 2024-11-20T09:01:48,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-20T09:01:48,797 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-20T09:01:48,800 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-20T09:01:48,806 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testScanAtomicity Thread=242 (was 241) - Thread LEAK? -, OpenFileDescriptor=455 (was 461), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=384 (was 412), ProcessCount=11 (was 11), AvailableMemoryMB=1988 (was 2239) 2024-11-20T09:01:48,815 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobGetAtomicity Thread=242, OpenFileDescriptor=455, MaxFileDescriptor=1048576, SystemLoadAverage=384, ProcessCount=11, AvailableMemoryMB=1987 2024-11-20T09:01:48,816 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-20T09:01:48,816 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-20T09:01:48,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] procedure2.ProcedureExecutor(1139): Stored pid=136, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-20T09:01:48,818 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=136, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-20T09:01:48,818 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:48,818 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 136 2024-11-20T09:01:48,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=136 2024-11-20T09:01:48,819 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=136, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-20T09:01:48,823 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742381_1557 (size=959) 2024-11-20T09:01:48,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=136 2024-11-20T09:01:49,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=136 2024-11-20T09:01:49,225 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 2efe576d1bb7feeeb5d5727034c1a290, NAME => 'TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a 2024-11-20T09:01:49,233 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742382_1558 (size=53) 2024-11-20T09:01:49,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=136 2024-11-20T09:01:49,634 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T09:01:49,634 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 2efe576d1bb7feeeb5d5727034c1a290, disabling compactions & flushes 2024-11-20T09:01:49,634 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290. 2024-11-20T09:01:49,634 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290. 2024-11-20T09:01:49,635 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290. after waiting 0 ms 2024-11-20T09:01:49,635 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290. 2024-11-20T09:01:49,635 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290. 2024-11-20T09:01:49,635 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 2efe576d1bb7feeeb5d5727034c1a290: Waiting for close lock at 1732093309634Disabling compacts and flushes for region at 1732093309634Disabling writes for close at 1732093309635 (+1 ms)Writing region close event to WAL at 1732093309635Closed at 1732093309635 2024-11-20T09:01:49,636 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=136, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-20T09:01:49,636 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732093309636"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732093309636"}]},"ts":"1732093309636"} 2024-11-20T09:01:49,637 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-20T09:01:49,638 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=136, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-20T09:01:49,638 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732093309638"}]},"ts":"1732093309638"} 2024-11-20T09:01:49,639 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-20T09:01:49,639 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=137, ppid=136, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2efe576d1bb7feeeb5d5727034c1a290, ASSIGN}] 2024-11-20T09:01:49,640 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=137, ppid=136, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2efe576d1bb7feeeb5d5727034c1a290, ASSIGN 2024-11-20T09:01:49,641 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=137, ppid=136, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=2efe576d1bb7feeeb5d5727034c1a290, ASSIGN; state=OFFLINE, location=be1eb2620e0e,34223,1732093159960; forceNewPlan=false, retain=false 2024-11-20T09:01:49,645 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-20T09:01:49,792 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=137 updating hbase:meta row=2efe576d1bb7feeeb5d5727034c1a290, regionState=OPENING, regionLocation=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:49,793 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=137, ppid=136, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=2efe576d1bb7feeeb5d5727034c1a290, ASSIGN because future has completed 2024-11-20T09:01:49,793 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=138, ppid=137, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960}] 2024-11-20T09:01:49,947 INFO [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290. 2024-11-20T09:01:49,947 DEBUG [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(7752): Opening region: {ENCODED => 2efe576d1bb7feeeb5d5727034c1a290, NAME => 'TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290.', STARTKEY => '', ENDKEY => ''} 2024-11-20T09:01:49,948 DEBUG [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 2efe576d1bb7feeeb5d5727034c1a290 2024-11-20T09:01:49,948 DEBUG [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T09:01:49,948 DEBUG [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(7794): checking encryption for 2efe576d1bb7feeeb5d5727034c1a290 2024-11-20T09:01:49,948 DEBUG [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(7797): checking classloading for 2efe576d1bb7feeeb5d5727034c1a290 2024-11-20T09:01:49,949 INFO [StoreOpener-2efe576d1bb7feeeb5d5727034c1a290-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 2efe576d1bb7feeeb5d5727034c1a290 2024-11-20T09:01:49,950 INFO [StoreOpener-2efe576d1bb7feeeb5d5727034c1a290-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2efe576d1bb7feeeb5d5727034c1a290 columnFamilyName A 2024-11-20T09:01:49,950 DEBUG [StoreOpener-2efe576d1bb7feeeb5d5727034c1a290-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:49,950 INFO [StoreOpener-2efe576d1bb7feeeb5d5727034c1a290-1 {}] regionserver.HStore(327): Store=2efe576d1bb7feeeb5d5727034c1a290/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T09:01:49,951 INFO [StoreOpener-2efe576d1bb7feeeb5d5727034c1a290-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 2efe576d1bb7feeeb5d5727034c1a290 2024-11-20T09:01:49,951 INFO [StoreOpener-2efe576d1bb7feeeb5d5727034c1a290-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2efe576d1bb7feeeb5d5727034c1a290 columnFamilyName B 2024-11-20T09:01:49,951 DEBUG [StoreOpener-2efe576d1bb7feeeb5d5727034c1a290-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:49,952 INFO [StoreOpener-2efe576d1bb7feeeb5d5727034c1a290-1 {}] regionserver.HStore(327): Store=2efe576d1bb7feeeb5d5727034c1a290/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T09:01:49,952 INFO [StoreOpener-2efe576d1bb7feeeb5d5727034c1a290-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 2efe576d1bb7feeeb5d5727034c1a290 2024-11-20T09:01:49,953 INFO [StoreOpener-2efe576d1bb7feeeb5d5727034c1a290-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2efe576d1bb7feeeb5d5727034c1a290 columnFamilyName C 2024-11-20T09:01:49,953 DEBUG [StoreOpener-2efe576d1bb7feeeb5d5727034c1a290-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:49,953 INFO [StoreOpener-2efe576d1bb7feeeb5d5727034c1a290-1 {}] regionserver.HStore(327): Store=2efe576d1bb7feeeb5d5727034c1a290/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T09:01:49,953 DEBUG [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(1038): replaying wal for 2efe576d1bb7feeeb5d5727034c1a290 2024-11-20T09:01:49,954 DEBUG [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290 2024-11-20T09:01:49,954 DEBUG [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290 2024-11-20T09:01:49,955 DEBUG [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(1048): stopping wal replay for 2efe576d1bb7feeeb5d5727034c1a290 2024-11-20T09:01:49,955 DEBUG [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(1060): Cleaning up temporary data for 2efe576d1bb7feeeb5d5727034c1a290 2024-11-20T09:01:49,955 DEBUG [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-20T09:01:49,956 DEBUG [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(1093): writing seq id for 2efe576d1bb7feeeb5d5727034c1a290 2024-11-20T09:01:49,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=136 2024-11-20T09:01:49,957 DEBUG [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T09:01:49,958 INFO [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(1114): Opened 2efe576d1bb7feeeb5d5727034c1a290; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62425996, jitterRate=-0.06978017091751099}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-20T09:01:49,958 DEBUG [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 2efe576d1bb7feeeb5d5727034c1a290 2024-11-20T09:01:49,958 DEBUG [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(1006): Region open journal for 2efe576d1bb7feeeb5d5727034c1a290: Running coprocessor pre-open hook at 1732093309948Writing region info on filesystem at 1732093309948Initializing all the Stores at 1732093309948Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732093309948Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732093309949 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732093309949Cleaning up temporary data from old regions at 1732093309955 (+6 ms)Running coprocessor post-open hooks at 1732093309958 (+3 ms)Region opened successfully at 1732093309958 2024-11-20T09:01:49,959 INFO [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., pid=138, masterSystemTime=1732093309945 2024-11-20T09:01:49,960 DEBUG [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290. 2024-11-20T09:01:49,960 INFO [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290. 2024-11-20T09:01:49,961 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=137 updating hbase:meta row=2efe576d1bb7feeeb5d5727034c1a290, regionState=OPEN, openSeqNum=2, regionLocation=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:49,962 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=138, ppid=137, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 because future has completed 2024-11-20T09:01:49,964 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=138, resume processing ppid=137 2024-11-20T09:01:49,964 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=138, ppid=137, state=SUCCESS, hasLock=false; OpenRegionProcedure 2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 in 169 msec 2024-11-20T09:01:49,965 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=137, resume processing ppid=136 2024-11-20T09:01:49,965 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=137, ppid=136, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2efe576d1bb7feeeb5d5727034c1a290, ASSIGN in 325 msec 2024-11-20T09:01:49,966 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=136, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-20T09:01:49,966 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732093309966"}]},"ts":"1732093309966"} 2024-11-20T09:01:49,967 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-20T09:01:49,967 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=136, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-20T09:01:49,968 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=136, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1510 sec 2024-11-20T09:01:50,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=136 2024-11-20T09:01:50,967 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-20T09:01:50,968 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@145ad34b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T09:01:50,968 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request be1eb2620e0e,44373,-1 for getting cluster id 2024-11-20T09:01:50,968 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T09:01:50,969 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'caf50035-b19e-4cfd-90ea-dca559866882' 2024-11-20T09:01:50,969 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T09:01:50,970 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "caf50035-b19e-4cfd-90ea-dca559866882" 2024-11-20T09:01:50,970 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3d18f35e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T09:01:50,970 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [be1eb2620e0e,44373,-1] 2024-11-20T09:01:50,970 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T09:01:50,970 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T09:01:50,971 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52312, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T09:01:50,971 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ffca22e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T09:01:50,972 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T09:01:50,972 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=be1eb2620e0e,34223,1732093159960, seqNum=-1] 2024-11-20T09:01:50,972 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T09:01:50,973 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53366, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T09:01:50,974 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-20T09:01:50,974 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncConnectionImpl(321): The fetched master address is be1eb2620e0e,44373,1732093159150 2024-11-20T09:01:50,974 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@284a86e4 2024-11-20T09:01:50,974 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-20T09:01:50,975 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52316, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-20T09:01:50,976 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-20T09:01:50,977 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-20T09:01:50,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] procedure2.ProcedureExecutor(1139): Stored pid=139, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-20T09:01:50,985 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742383_1559 (size=995) 2024-11-20T09:01:51,386 DEBUG [PEWorker-1 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-11-20T09:01:51,387 INFO [PEWorker-1 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-11-20T09:01:51,389 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=140, ppid=139, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-20T09:01:51,391 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2efe576d1bb7feeeb5d5727034c1a290, REOPEN/MOVE}] 2024-11-20T09:01:51,391 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2efe576d1bb7feeeb5d5727034c1a290, REOPEN/MOVE 2024-11-20T09:01:51,392 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=141 updating hbase:meta row=2efe576d1bb7feeeb5d5727034c1a290, regionState=CLOSING, regionLocation=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:51,393 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=2efe576d1bb7feeeb5d5727034c1a290, REOPEN/MOVE because future has completed 2024-11-20T09:01:51,393 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-20T09:01:51,393 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=142, ppid=141, state=RUNNABLE, hasLock=false; CloseRegionProcedure 2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960}] 2024-11-20T09:01:51,545 INFO [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] handler.UnassignRegionHandler(122): Close 2efe576d1bb7feeeb5d5727034c1a290 2024-11-20T09:01:51,545 DEBUG [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-20T09:01:51,545 DEBUG [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1722): Closing 2efe576d1bb7feeeb5d5727034c1a290, disabling compactions & flushes 2024-11-20T09:01:51,545 INFO [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290. 2024-11-20T09:01:51,545 DEBUG [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290. 2024-11-20T09:01:51,545 DEBUG [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290. after waiting 0 ms 2024-11-20T09:01:51,545 DEBUG [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290. 2024-11-20T09:01:51,548 DEBUG [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-20T09:01:51,549 INFO [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290. 2024-11-20T09:01:51,549 DEBUG [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1676): Region close journal for 2efe576d1bb7feeeb5d5727034c1a290: Waiting for close lock at 1732093311545Running coprocessor pre-close hooks at 1732093311545Disabling compacts and flushes for region at 1732093311545Disabling writes for close at 1732093311545Writing region close event to WAL at 1732093311545Running coprocessor post-close hooks at 1732093311549 (+4 ms)Closed at 1732093311549 2024-11-20T09:01:51,549 WARN [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegionServer(3299): Not adding moved region record: 2efe576d1bb7feeeb5d5727034c1a290 to self. 2024-11-20T09:01:51,550 INFO [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] handler.UnassignRegionHandler(157): Closed 2efe576d1bb7feeeb5d5727034c1a290 2024-11-20T09:01:51,551 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=141 updating hbase:meta row=2efe576d1bb7feeeb5d5727034c1a290, regionState=CLOSED 2024-11-20T09:01:51,552 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=142, ppid=141, state=RUNNABLE, hasLock=false; CloseRegionProcedure 2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 because future has completed 2024-11-20T09:01:51,554 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=142, resume processing ppid=141 2024-11-20T09:01:51,554 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=142, ppid=141, state=SUCCESS, hasLock=false; CloseRegionProcedure 2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 in 159 msec 2024-11-20T09:01:51,554 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=2efe576d1bb7feeeb5d5727034c1a290, REOPEN/MOVE; state=CLOSED, location=be1eb2620e0e,34223,1732093159960; forceNewPlan=false, retain=true 2024-11-20T09:01:51,705 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=141 updating hbase:meta row=2efe576d1bb7feeeb5d5727034c1a290, regionState=OPENING, regionLocation=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:51,706 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=2efe576d1bb7feeeb5d5727034c1a290, REOPEN/MOVE because future has completed 2024-11-20T09:01:51,706 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=143, ppid=141, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960}] 2024-11-20T09:01:51,860 INFO [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290. 2024-11-20T09:01:51,861 DEBUG [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(7752): Opening region: {ENCODED => 2efe576d1bb7feeeb5d5727034c1a290, NAME => 'TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290.', STARTKEY => '', ENDKEY => ''} 2024-11-20T09:01:51,861 DEBUG [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 2efe576d1bb7feeeb5d5727034c1a290 2024-11-20T09:01:51,861 DEBUG [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T09:01:51,861 DEBUG [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(7794): checking encryption for 2efe576d1bb7feeeb5d5727034c1a290 2024-11-20T09:01:51,861 DEBUG [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(7797): checking classloading for 2efe576d1bb7feeeb5d5727034c1a290 2024-11-20T09:01:51,862 INFO [StoreOpener-2efe576d1bb7feeeb5d5727034c1a290-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 2efe576d1bb7feeeb5d5727034c1a290 2024-11-20T09:01:51,863 INFO [StoreOpener-2efe576d1bb7feeeb5d5727034c1a290-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2efe576d1bb7feeeb5d5727034c1a290 columnFamilyName A 2024-11-20T09:01:51,864 DEBUG [StoreOpener-2efe576d1bb7feeeb5d5727034c1a290-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:51,865 INFO [StoreOpener-2efe576d1bb7feeeb5d5727034c1a290-1 {}] regionserver.HStore(327): Store=2efe576d1bb7feeeb5d5727034c1a290/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T09:01:51,865 INFO [StoreOpener-2efe576d1bb7feeeb5d5727034c1a290-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 2efe576d1bb7feeeb5d5727034c1a290 2024-11-20T09:01:51,865 INFO [StoreOpener-2efe576d1bb7feeeb5d5727034c1a290-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2efe576d1bb7feeeb5d5727034c1a290 columnFamilyName B 2024-11-20T09:01:51,865 DEBUG [StoreOpener-2efe576d1bb7feeeb5d5727034c1a290-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:51,866 INFO [StoreOpener-2efe576d1bb7feeeb5d5727034c1a290-1 {}] regionserver.HStore(327): Store=2efe576d1bb7feeeb5d5727034c1a290/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T09:01:51,866 INFO [StoreOpener-2efe576d1bb7feeeb5d5727034c1a290-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 2efe576d1bb7feeeb5d5727034c1a290 2024-11-20T09:01:51,866 INFO [StoreOpener-2efe576d1bb7feeeb5d5727034c1a290-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2efe576d1bb7feeeb5d5727034c1a290 columnFamilyName C 2024-11-20T09:01:51,866 DEBUG [StoreOpener-2efe576d1bb7feeeb5d5727034c1a290-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:51,867 INFO [StoreOpener-2efe576d1bb7feeeb5d5727034c1a290-1 {}] regionserver.HStore(327): Store=2efe576d1bb7feeeb5d5727034c1a290/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T09:01:51,867 DEBUG [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(1038): replaying wal for 2efe576d1bb7feeeb5d5727034c1a290 2024-11-20T09:01:51,867 DEBUG [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290 2024-11-20T09:01:51,868 DEBUG [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290 2024-11-20T09:01:51,869 DEBUG [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(1048): stopping wal replay for 2efe576d1bb7feeeb5d5727034c1a290 2024-11-20T09:01:51,869 DEBUG [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(1060): Cleaning up temporary data for 2efe576d1bb7feeeb5d5727034c1a290 2024-11-20T09:01:51,869 DEBUG [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-20T09:01:51,870 DEBUG [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(1093): writing seq id for 2efe576d1bb7feeeb5d5727034c1a290 2024-11-20T09:01:51,870 INFO [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(1114): Opened 2efe576d1bb7feeeb5d5727034c1a290; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63007443, jitterRate=-0.06111593544483185}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-20T09:01:51,871 DEBUG [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 2efe576d1bb7feeeb5d5727034c1a290 2024-11-20T09:01:51,871 DEBUG [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(1006): Region open journal for 2efe576d1bb7feeeb5d5727034c1a290: Running coprocessor pre-open hook at 1732093311861Writing region info on filesystem at 1732093311861Initializing all the Stores at 1732093311862 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732093311862Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732093311862Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732093311862Cleaning up temporary data from old regions at 1732093311869 (+7 ms)Running coprocessor post-open hooks at 1732093311871 (+2 ms)Region opened successfully at 1732093311871 2024-11-20T09:01:51,872 INFO [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., pid=143, masterSystemTime=1732093311858 2024-11-20T09:01:51,873 DEBUG [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290. 2024-11-20T09:01:51,873 INFO [RS_OPEN_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290. 2024-11-20T09:01:51,874 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=141 updating hbase:meta row=2efe576d1bb7feeeb5d5727034c1a290, regionState=OPEN, openSeqNum=5, regionLocation=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:51,875 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=143, ppid=141, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 because future has completed 2024-11-20T09:01:51,877 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=143, resume processing ppid=141 2024-11-20T09:01:51,877 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=143, ppid=141, state=SUCCESS, hasLock=false; OpenRegionProcedure 2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 in 170 msec 2024-11-20T09:01:51,878 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=141, resume processing ppid=140 2024-11-20T09:01:51,878 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=141, ppid=140, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2efe576d1bb7feeeb5d5727034c1a290, REOPEN/MOVE in 487 msec 2024-11-20T09:01:51,880 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=140, resume processing ppid=139 2024-11-20T09:01:51,880 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=140, ppid=139, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 489 msec 2024-11-20T09:01:51,881 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=139, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 903 msec 2024-11-20T09:01:51,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-20T09:01:51,882 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-20T09:01:51,883 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@143fb08, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T09:01:51,883 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request be1eb2620e0e,44373,-1 for getting cluster id 2024-11-20T09:01:51,883 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T09:01:51,884 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'caf50035-b19e-4cfd-90ea-dca559866882' 2024-11-20T09:01:51,884 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T09:01:51,884 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "caf50035-b19e-4cfd-90ea-dca559866882" 2024-11-20T09:01:51,884 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@15ec9e43, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T09:01:51,884 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [be1eb2620e0e,44373,-1] 2024-11-20T09:01:51,884 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T09:01:51,884 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T09:01:51,885 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52340, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T09:01:51,886 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@470b7f2c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T09:01:51,886 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40503a36, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T09:01:51,886 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request be1eb2620e0e,44373,-1 for getting cluster id 2024-11-20T09:01:51,886 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T09:01:51,887 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'caf50035-b19e-4cfd-90ea-dca559866882' 2024-11-20T09:01:51,887 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T09:01:51,887 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "caf50035-b19e-4cfd-90ea-dca559866882" 2024-11-20T09:01:51,887 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ab79160, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T09:01:51,887 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [be1eb2620e0e,44373,-1] 2024-11-20T09:01:51,887 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T09:01:51,887 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T09:01:51,888 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52370, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T09:01:51,888 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43dc5a3b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T09:01:51,889 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52f72e1a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T09:01:51,889 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request be1eb2620e0e,44373,-1 for getting cluster id 2024-11-20T09:01:51,889 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T09:01:51,889 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'caf50035-b19e-4cfd-90ea-dca559866882' 2024-11-20T09:01:51,890 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T09:01:51,890 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "caf50035-b19e-4cfd-90ea-dca559866882" 2024-11-20T09:01:51,890 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@66f9c4c1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T09:01:51,890 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [be1eb2620e0e,44373,-1] 2024-11-20T09:01:51,890 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T09:01:51,890 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T09:01:51,891 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52384, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T09:01:51,891 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47792f21, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T09:01:51,892 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47173f60, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T09:01:51,892 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request be1eb2620e0e,44373,-1 for getting cluster id 2024-11-20T09:01:51,892 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T09:01:51,892 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'caf50035-b19e-4cfd-90ea-dca559866882' 2024-11-20T09:01:51,892 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T09:01:51,892 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "caf50035-b19e-4cfd-90ea-dca559866882" 2024-11-20T09:01:51,892 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2245a9db, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T09:01:51,892 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [be1eb2620e0e,44373,-1] 2024-11-20T09:01:51,893 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T09:01:51,893 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T09:01:51,893 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52392, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T09:01:51,894 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1400d4e1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T09:01:51,894 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13700528, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T09:01:51,894 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request be1eb2620e0e,44373,-1 for getting cluster id 2024-11-20T09:01:51,894 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T09:01:51,895 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'caf50035-b19e-4cfd-90ea-dca559866882' 2024-11-20T09:01:51,895 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T09:01:51,895 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "caf50035-b19e-4cfd-90ea-dca559866882" 2024-11-20T09:01:51,895 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@53d02da2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T09:01:51,895 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [be1eb2620e0e,44373,-1] 2024-11-20T09:01:51,895 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T09:01:51,895 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T09:01:51,896 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52412, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T09:01:51,896 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@48be2747, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T09:01:51,897 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3666b173, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T09:01:51,897 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request be1eb2620e0e,44373,-1 for getting cluster id 2024-11-20T09:01:51,897 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T09:01:51,897 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'caf50035-b19e-4cfd-90ea-dca559866882' 2024-11-20T09:01:51,898 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T09:01:51,898 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "caf50035-b19e-4cfd-90ea-dca559866882" 2024-11-20T09:01:51,898 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59d22141, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T09:01:51,898 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [be1eb2620e0e,44373,-1] 2024-11-20T09:01:51,898 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T09:01:51,898 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T09:01:51,899 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52430, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T09:01:51,899 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@9f64930, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T09:01:51,900 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77ff027c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T09:01:51,900 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request be1eb2620e0e,44373,-1 for getting cluster id 2024-11-20T09:01:51,900 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T09:01:51,900 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'caf50035-b19e-4cfd-90ea-dca559866882' 2024-11-20T09:01:51,900 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T09:01:51,901 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "caf50035-b19e-4cfd-90ea-dca559866882" 2024-11-20T09:01:51,901 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@9730f93, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T09:01:51,901 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [be1eb2620e0e,44373,-1] 2024-11-20T09:01:51,901 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T09:01:51,901 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T09:01:51,901 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52444, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T09:01:51,902 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6febff20, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T09:01:51,902 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2637386d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T09:01:51,902 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request be1eb2620e0e,44373,-1 for getting cluster id 2024-11-20T09:01:51,902 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T09:01:51,903 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'caf50035-b19e-4cfd-90ea-dca559866882' 2024-11-20T09:01:51,903 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T09:01:51,903 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "caf50035-b19e-4cfd-90ea-dca559866882" 2024-11-20T09:01:51,903 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@20fb2323, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T09:01:51,903 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [be1eb2620e0e,44373,-1] 2024-11-20T09:01:51,904 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T09:01:51,904 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T09:01:51,904 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52462, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T09:01:51,905 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32abf31b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T09:01:51,905 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2455dde6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T09:01:51,905 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request be1eb2620e0e,44373,-1 for getting cluster id 2024-11-20T09:01:51,905 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T09:01:51,906 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'caf50035-b19e-4cfd-90ea-dca559866882' 2024-11-20T09:01:51,906 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T09:01:51,906 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "caf50035-b19e-4cfd-90ea-dca559866882" 2024-11-20T09:01:51,906 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@664bce78, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T09:01:51,906 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [be1eb2620e0e,44373,-1] 2024-11-20T09:01:51,906 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T09:01:51,907 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T09:01:51,907 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52488, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T09:01:51,907 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@581e2813, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T09:01:51,908 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b93642c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T09:01:51,908 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request be1eb2620e0e,44373,-1 for getting cluster id 2024-11-20T09:01:51,908 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T09:01:51,908 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'caf50035-b19e-4cfd-90ea-dca559866882' 2024-11-20T09:01:51,908 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T09:01:51,908 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "caf50035-b19e-4cfd-90ea-dca559866882" 2024-11-20T09:01:51,909 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@594ef3f7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T09:01:51,909 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [be1eb2620e0e,44373,-1] 2024-11-20T09:01:51,909 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T09:01:51,909 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T09:01:51,909 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52506, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T09:01:51,910 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@327dc9f5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T09:01:51,910 DEBUG [Thread-2487 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T09:01:51,910 DEBUG [Thread-2500 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T09:01:51,910 DEBUG [Thread-2498 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T09:01:51,910 DEBUG [Thread-2491 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T09:01:51,911 DEBUG [Thread-2502 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T09:01:51,911 DEBUG [Thread-2489 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T09:01:51,911 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=be1eb2620e0e,34223,1732093159960, seqNum=-1] 2024-11-20T09:01:51,911 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=be1eb2620e0e,34223,1732093159960, seqNum=-1] 2024-11-20T09:01:51,911 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=be1eb2620e0e,34223,1732093159960, seqNum=-1] 2024-11-20T09:01:51,911 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=be1eb2620e0e,34223,1732093159960, seqNum=-1] 2024-11-20T09:01:51,911 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=be1eb2620e0e,34223,1732093159960, seqNum=-1] 2024-11-20T09:01:51,911 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T09:01:51,911 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T09:01:51,911 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T09:01:51,911 DEBUG [Thread-2504 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T09:01:51,911 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T09:01:51,912 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T09:01:51,912 DEBUG [Thread-2496 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T09:01:51,912 DEBUG [Thread-2485 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T09:01:51,912 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=be1eb2620e0e,34223,1732093159960, seqNum=-1] 2024-11-20T09:01:51,912 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53372, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T09:01:51,912 DEBUG [Thread-2493 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T09:01:51,912 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T09:01:51,913 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T09:01:51,913 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=be1eb2620e0e,34223,1732093159960, seqNum=-1] 2024-11-20T09:01:51,913 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53418, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T09:01:51,913 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=be1eb2620e0e,34223,1732093159960, seqNum=-1] 2024-11-20T09:01:51,913 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T09:01:51,913 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=be1eb2620e0e,34223,1732093159960, seqNum=-1] 2024-11-20T09:01:51,913 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T09:01:51,913 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T09:01:51,913 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53388, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T09:01:51,913 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53390, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T09:01:51,914 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53434, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T09:01:51,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] procedure2.ProcedureExecutor(1139): Stored pid=144, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=144, table=TestAcidGuarantees 2024-11-20T09:01:51,914 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53402, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T09:01:51,914 DEBUG [Thread-2493 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=be1eb2620e0e,34223,1732093159960, seqNum=-1] 2024-11-20T09:01:51,914 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53438, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T09:01:51,914 DEBUG [Thread-2493 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T09:01:51,915 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53442, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T09:01:51,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-20T09:01:51,915 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53446, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T09:01:51,915 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5] 2024-11-20T09:01:51,915 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5] 2024-11-20T09:01:51,915 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5] 2024-11-20T09:01:51,915 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5] 2024-11-20T09:01:51,916 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5] 2024-11-20T09:01:51,916 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53462, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T09:01:51,916 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=144, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=144, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T09:01:51,917 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5] 2024-11-20T09:01:51,917 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5] 2024-11-20T09:01:51,917 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=144, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=144, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T09:01:51,917 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=145, ppid=144, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T09:01:51,917 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5] 2024-11-20T09:01:51,918 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5] 2024-11-20T09:01:51,918 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5] 2024-11-20T09:01:51,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(8855): Flush requested on 2efe576d1bb7feeeb5d5727034c1a290 2024-11-20T09:01:51,924 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T09:01:51,924 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2efe576d1bb7feeeb5d5727034c1a290 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-20T09:01:51,947 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411208852ea0b2cb448159ab7b33d76f4f98d_2efe576d1bb7feeeb5d5727034c1a290 is 50, key is test_row_0/A:col10/1732093311921/Put/seqid=0 2024-11-20T09:01:51,950 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742384_1560 (size=12154) 2024-11-20T09:01:51,981 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:51,981 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:51,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53418 deadline: 1732093371953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:51,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53462 deadline: 1732093371952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:51,981 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:51,981 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:51,981 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:51,981 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:51,981 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:51,981 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:51,985 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:51,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53372 deadline: 1732093371981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:51,985 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:51,985 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:51,985 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:51,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53446 deadline: 1732093371981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:51,985 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:51,985 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:51,985 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:51,986 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:51,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53442 deadline: 1732093371981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:51,986 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:51,986 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:51,986 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:51,986 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:52,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-20T09:01:52,071 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-20T09:01:52,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290. 2024-11-20T09:01:52,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290. as already flushing 2024-11-20T09:01:52,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290. 2024-11-20T09:01:52,071 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => 2efe576d1bb7feeeb5d5727034c1a290, NAME => 'TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:52,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => 2efe576d1bb7feeeb5d5727034c1a290, NAME => 'TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:52,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2efe576d1bb7feeeb5d5727034c1a290, NAME => 'TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2efe576d1bb7feeeb5d5727034c1a290, NAME => 'TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:52,089 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:52,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53418 deadline: 1732093372086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:52,089 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:52,089 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:52,089 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:52,090 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:52,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53462 deadline: 1732093372087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:52,090 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:52,090 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:52,090 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:52,098 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:52,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53372 deadline: 1732093372097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:52,098 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:52,098 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:52,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53442 deadline: 1732093372097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:52,098 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:52,098 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:52,099 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:52,099 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:52,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53446 deadline: 1732093372097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:52,099 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:52,099 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:52,099 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:52,099 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:52,099 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:52,224 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-20T09:01:52,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290. 2024-11-20T09:01:52,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290. as already flushing 2024-11-20T09:01:52,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290. 2024-11-20T09:01:52,224 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => 2efe576d1bb7feeeb5d5727034c1a290, NAME => 'TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:52,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => 2efe576d1bb7feeeb5d5727034c1a290, NAME => 'TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:52,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2efe576d1bb7feeeb5d5727034c1a290, NAME => 'TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2efe576d1bb7feeeb5d5727034c1a290, NAME => 'TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:52,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-20T09:01:52,298 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:52,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53418 deadline: 1732093372297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:52,299 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:52,298 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:52,299 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:52,299 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:52,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53462 deadline: 1732093372297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:52,299 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:52,299 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:52,299 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:52,310 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:52,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53372 deadline: 1732093372307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:52,310 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:52,310 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:52,310 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:52,310 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:52,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53442 deadline: 1732093372307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:52,311 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:52,311 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:52,311 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:52,311 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:52,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53446 deadline: 1732093372307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:52,311 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:52,311 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:52,311 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:52,351 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:52,354 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411208852ea0b2cb448159ab7b33d76f4f98d_2efe576d1bb7feeeb5d5727034c1a290 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411208852ea0b2cb448159ab7b33d76f4f98d_2efe576d1bb7feeeb5d5727034c1a290 2024-11-20T09:01:52,355 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/.tmp/A/204bd333446043fa8a8c9beb08a5f232, store: [table=TestAcidGuarantees family=A region=2efe576d1bb7feeeb5d5727034c1a290] 2024-11-20T09:01:52,355 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/.tmp/A/204bd333446043fa8a8c9beb08a5f232 is 175, key is test_row_0/A:col10/1732093311921/Put/seqid=0 2024-11-20T09:01:52,359 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742385_1561 (size=30955) 2024-11-20T09:01:52,377 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-20T09:01:52,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290. 2024-11-20T09:01:52,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290. as already flushing 2024-11-20T09:01:52,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290. 2024-11-20T09:01:52,377 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => 2efe576d1bb7feeeb5d5727034c1a290, NAME => 'TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:52,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => 2efe576d1bb7feeeb5d5727034c1a290, NAME => 'TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:52,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2efe576d1bb7feeeb5d5727034c1a290, NAME => 'TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2efe576d1bb7feeeb5d5727034c1a290, NAME => 'TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:52,529 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-20T09:01:52,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290. 2024-11-20T09:01:52,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290. as already flushing 2024-11-20T09:01:52,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290. 2024-11-20T09:01:52,530 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => 2efe576d1bb7feeeb5d5727034c1a290, NAME => 'TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:52,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => 2efe576d1bb7feeeb5d5727034c1a290, NAME => 'TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:52,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2efe576d1bb7feeeb5d5727034c1a290, NAME => 'TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2efe576d1bb7feeeb5d5727034c1a290, NAME => 'TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:52,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-20T09:01:52,609 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:52,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53418 deadline: 1732093372607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:52,609 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:52,609 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:52,609 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:52,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53462 deadline: 1732093372607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:52,609 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:52,610 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:52,610 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:52,610 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:52,618 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:52,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53442 deadline: 1732093372617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:52,618 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:52,618 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:52,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53372 deadline: 1732093372617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:52,618 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:52,618 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:52,618 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:52,618 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:52,618 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:52,618 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:52,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53446 deadline: 1732093372617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:52,619 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:52,619 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:52,619 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:52,682 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-20T09:01:52,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290. 2024-11-20T09:01:52,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290. as already flushing 2024-11-20T09:01:52,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290. 2024-11-20T09:01:52,683 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => 2efe576d1bb7feeeb5d5727034c1a290, NAME => 'TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:52,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => 2efe576d1bb7feeeb5d5727034c1a290, NAME => 'TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:52,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2efe576d1bb7feeeb5d5727034c1a290, NAME => 'TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2efe576d1bb7feeeb5d5727034c1a290, NAME => 'TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:52,760 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/.tmp/A/204bd333446043fa8a8c9beb08a5f232 2024-11-20T09:01:52,783 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/.tmp/B/919cbddd2375495abf2a7ae69c21d872 is 50, key is test_row_0/B:col10/1732093311921/Put/seqid=0 2024-11-20T09:01:52,786 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742386_1562 (size=12001) 2024-11-20T09:01:52,835 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-20T09:01:52,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290. 2024-11-20T09:01:52,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290. as already flushing 2024-11-20T09:01:52,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290. 2024-11-20T09:01:52,836 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => 2efe576d1bb7feeeb5d5727034c1a290, NAME => 'TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:52,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => 2efe576d1bb7feeeb5d5727034c1a290, NAME => 'TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:52,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2efe576d1bb7feeeb5d5727034c1a290, NAME => 'TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2efe576d1bb7feeeb5d5727034c1a290, NAME => 'TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:52,988 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-20T09:01:52,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290. 2024-11-20T09:01:52,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290. as already flushing 2024-11-20T09:01:52,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290. 2024-11-20T09:01:52,989 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => 2efe576d1bb7feeeb5d5727034c1a290, NAME => 'TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:52,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => 2efe576d1bb7feeeb5d5727034c1a290, NAME => 'TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:52,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2efe576d1bb7feeeb5d5727034c1a290, NAME => 'TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2efe576d1bb7feeeb5d5727034c1a290, NAME => 'TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:53,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-20T09:01:53,119 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:53,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53418 deadline: 1732093373117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:53,120 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:53,120 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:53,120 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:53,120 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:53,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53462 deadline: 1732093373117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:53,120 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:53,120 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:53,120 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:53,130 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:53,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53442 deadline: 1732093373127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:53,130 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:53,130 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:53,130 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:53,130 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:53,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53372 deadline: 1732093373127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:53,130 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:53,130 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:53,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53446 deadline: 1732093373127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:53,130 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:53,130 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:53,131 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:53,131 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:53,131 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:53,141 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-20T09:01:53,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290. 2024-11-20T09:01:53,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290. as already flushing 2024-11-20T09:01:53,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290. 2024-11-20T09:01:53,141 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => 2efe576d1bb7feeeb5d5727034c1a290, NAME => 'TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:53,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => 2efe576d1bb7feeeb5d5727034c1a290, NAME => 'TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:53,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2efe576d1bb7feeeb5d5727034c1a290, NAME => 'TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2efe576d1bb7feeeb5d5727034c1a290, NAME => 'TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:53,187 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/.tmp/B/919cbddd2375495abf2a7ae69c21d872 2024-11-20T09:01:53,207 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/.tmp/C/bfd37f541ac94c37a67737eca6a42199 is 50, key is test_row_0/C:col10/1732093311921/Put/seqid=0 2024-11-20T09:01:53,210 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742387_1563 (size=12001) 2024-11-20T09:01:53,211 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/.tmp/C/bfd37f541ac94c37a67737eca6a42199 2024-11-20T09:01:53,214 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/.tmp/A/204bd333446043fa8a8c9beb08a5f232 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/A/204bd333446043fa8a8c9beb08a5f232 2024-11-20T09:01:53,217 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/A/204bd333446043fa8a8c9beb08a5f232, entries=150, sequenceid=15, filesize=30.2 K 2024-11-20T09:01:53,218 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/.tmp/B/919cbddd2375495abf2a7ae69c21d872 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/B/919cbddd2375495abf2a7ae69c21d872 2024-11-20T09:01:53,221 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/B/919cbddd2375495abf2a7ae69c21d872, entries=150, sequenceid=15, filesize=11.7 K 2024-11-20T09:01:53,221 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/.tmp/C/bfd37f541ac94c37a67737eca6a42199 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/C/bfd37f541ac94c37a67737eca6a42199 2024-11-20T09:01:53,225 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/C/bfd37f541ac94c37a67737eca6a42199, entries=150, sequenceid=15, filesize=11.7 K 2024-11-20T09:01:53,225 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=161.02 KB/164880 for 2efe576d1bb7feeeb5d5727034c1a290 in 1301ms, sequenceid=15, compaction requested=false 2024-11-20T09:01:53,225 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2efe576d1bb7feeeb5d5727034c1a290: 2024-11-20T09:01:53,294 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-20T09:01:53,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290. 2024-11-20T09:01:53,294 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2902): Flushing 2efe576d1bb7feeeb5d5727034c1a290 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-20T09:01:53,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112028ed1a493a1d4664967b3475e3f105e7_2efe576d1bb7feeeb5d5727034c1a290 is 50, key is test_row_0/A:col10/1732093311951/Put/seqid=0 2024-11-20T09:01:53,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742388_1564 (size=12154) 2024-11-20T09:01:53,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,311 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112028ed1a493a1d4664967b3475e3f105e7_2efe576d1bb7feeeb5d5727034c1a290 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112028ed1a493a1d4664967b3475e3f105e7_2efe576d1bb7feeeb5d5727034c1a290 2024-11-20T09:01:53,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/.tmp/A/737b2ab810cd4f8d887134abda2f5634, store: [table=TestAcidGuarantees family=A region=2efe576d1bb7feeeb5d5727034c1a290] 2024-11-20T09:01:53,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/.tmp/A/737b2ab810cd4f8d887134abda2f5634 is 175, key is test_row_0/A:col10/1732093311951/Put/seqid=0 2024-11-20T09:01:53,316 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742389_1565 (size=30955) 2024-11-20T09:01:53,316 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=42, memsize=53.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/.tmp/A/737b2ab810cd4f8d887134abda2f5634 2024-11-20T09:01:53,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/.tmp/B/cdd4ea03fc7b4a1e8cd7e4d5d2a62046 is 50, key is test_row_0/B:col10/1732093311951/Put/seqid=0 2024-11-20T09:01:53,335 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742390_1566 (size=12001) 2024-11-20T09:01:53,735 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=42 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/.tmp/B/cdd4ea03fc7b4a1e8cd7e4d5d2a62046 2024-11-20T09:01:53,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/.tmp/C/22dac69b05d8410a9ca31ddc8782e016 is 50, key is test_row_0/C:col10/1732093311951/Put/seqid=0 2024-11-20T09:01:53,745 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742391_1567 (size=12001) 2024-11-20T09:01:53,746 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=42 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/.tmp/C/22dac69b05d8410a9ca31ddc8782e016 2024-11-20T09:01:53,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/.tmp/A/737b2ab810cd4f8d887134abda2f5634 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/A/737b2ab810cd4f8d887134abda2f5634 2024-11-20T09:01:53,753 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/A/737b2ab810cd4f8d887134abda2f5634, entries=150, sequenceid=42, filesize=30.2 K 2024-11-20T09:01:53,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/.tmp/B/cdd4ea03fc7b4a1e8cd7e4d5d2a62046 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/B/cdd4ea03fc7b4a1e8cd7e4d5d2a62046 2024-11-20T09:01:53,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,756 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/B/cdd4ea03fc7b4a1e8cd7e4d5d2a62046, entries=150, sequenceid=42, filesize=11.7 K 2024-11-20T09:01:53,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/.tmp/C/22dac69b05d8410a9ca31ddc8782e016 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/C/22dac69b05d8410a9ca31ddc8782e016 2024-11-20T09:01:53,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,758 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-20T09:01:53,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,762 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/C/22dac69b05d8410a9ca31ddc8782e016, entries=150, sequenceid=42, filesize=11.7 K 2024-11-20T09:01:53,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,762 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=0 B/0 for 2efe576d1bb7feeeb5d5727034c1a290 in 468ms, sequenceid=42, compaction requested=false 2024-11-20T09:01:53,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2603): Flush status journal for 2efe576d1bb7feeeb5d5727034c1a290: 2024-11-20T09:01:53,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290. 2024-11-20T09:01:53,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=145 2024-11-20T09:01:53,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.HMaster(4169): Remote procedure done, pid=145 2024-11-20T09:01:53,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,766 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=145, resume processing ppid=144 2024-11-20T09:01:53,766 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=145, ppid=144, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8470 sec 2024-11-20T09:01:53,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,768 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=144, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=144, table=TestAcidGuarantees in 1.8540 sec 2024-11-20T09:01:53,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:53,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-20T09:01:54,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,057 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T09:01:54,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,057 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T09:01:54,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] procedure2.ProcedureExecutor(1139): Stored pid=146, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees 2024-11-20T09:01:54,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-20T09:01:54,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,060 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=146, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T09:01:54,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,060 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=146, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T09:01:54,061 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=147, ppid=146, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T09:01:54,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-20T09:01:54,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(8855): Flush requested on 2efe576d1bb7feeeb5d5727034c1a290 2024-11-20T09:01:54,175 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T09:01:54,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,175 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2efe576d1bb7feeeb5d5727034c1a290 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-20T09:01:54,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,183 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120eca9676d4eea439b8ddf07c5f3dd10ec_2efe576d1bb7feeeb5d5727034c1a290 is 50, key is test_row_0/A:col10/1732093314143/Put/seqid=0 2024-11-20T09:01:54,207 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742392_1568 (size=24358) 2024-11-20T09:01:54,213 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-20T09:01:54,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290. 2024-11-20T09:01:54,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290. as already flushing 2024-11-20T09:01:54,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290. 2024-11-20T09:01:54,213 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 2efe576d1bb7feeeb5d5727034c1a290, NAME => 'TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:54,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 2efe576d1bb7feeeb5d5727034c1a290, NAME => 'TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:54,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2efe576d1bb7feeeb5d5727034c1a290, NAME => 'TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2efe576d1bb7feeeb5d5727034c1a290, NAME => 'TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:54,250 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:54,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53418 deadline: 1732093374244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:54,250 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:54,250 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:54,250 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:54,251 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:54,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53442 deadline: 1732093374246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:54,252 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:54,252 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:54,252 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:54,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53372 deadline: 1732093374246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:54,252 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:54,252 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:54,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53462 deadline: 1732093374247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:54,252 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:54,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53446 deadline: 1732093374247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:54,253 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:54,253 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:54,253 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:54,253 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:54,253 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:54,253 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:54,253 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:54,253 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:54,253 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:54,359 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:54,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53418 deadline: 1732093374356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:54,359 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:54,359 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:54,359 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:54,359 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:54,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53442 deadline: 1732093374357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:54,359 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:54,360 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:54,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53446 deadline: 1732093374357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:54,360 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:54,360 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:54,360 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:54,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53372 deadline: 1732093374357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:54,360 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:54,360 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:54,360 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:54,360 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:54,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53462 deadline: 1732093374357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:54,360 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:54,360 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:54,360 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:54,360 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:54,360 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:54,360 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:54,366 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-20T09:01:54,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290. 2024-11-20T09:01:54,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290. as already flushing 2024-11-20T09:01:54,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290. 2024-11-20T09:01:54,366 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 2efe576d1bb7feeeb5d5727034c1a290, NAME => 'TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:54,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 2efe576d1bb7feeeb5d5727034c1a290, NAME => 'TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:54,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2efe576d1bb7feeeb5d5727034c1a290, NAME => 'TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2efe576d1bb7feeeb5d5727034c1a290, NAME => 'TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:54,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-20T09:01:54,519 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-20T09:01:54,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290. 2024-11-20T09:01:54,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290. as already flushing 2024-11-20T09:01:54,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290. 2024-11-20T09:01:54,519 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 2efe576d1bb7feeeb5d5727034c1a290, NAME => 'TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:54,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 2efe576d1bb7feeeb5d5727034c1a290, NAME => 'TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:54,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2efe576d1bb7feeeb5d5727034c1a290, NAME => 'TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2efe576d1bb7feeeb5d5727034c1a290, NAME => 'TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:54,570 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:54,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53418 deadline: 1732093374567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:54,570 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:54,570 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:54,570 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:54,570 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:54,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53442 deadline: 1732093374567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:54,570 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:54,571 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:54,571 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:54,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53446 deadline: 1732093374567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:54,571 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:54,571 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:54,571 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:54,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53372 deadline: 1732093374567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:54,571 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:54,571 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:54,571 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:54,571 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:54,571 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:54,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53462 deadline: 1732093374567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:54,571 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:54,571 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:54,571 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:54,571 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:54,608 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:54,611 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120eca9676d4eea439b8ddf07c5f3dd10ec_2efe576d1bb7feeeb5d5727034c1a290 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120eca9676d4eea439b8ddf07c5f3dd10ec_2efe576d1bb7feeeb5d5727034c1a290 2024-11-20T09:01:54,612 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/.tmp/A/763115a23ee04e84a31acd8f405995e7, store: [table=TestAcidGuarantees family=A region=2efe576d1bb7feeeb5d5727034c1a290] 2024-11-20T09:01:54,613 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/.tmp/A/763115a23ee04e84a31acd8f405995e7 is 175, key is test_row_0/A:col10/1732093314143/Put/seqid=0 2024-11-20T09:01:54,614 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742393_1569 (size=73994) 2024-11-20T09:01:54,672 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-20T09:01:54,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290. 2024-11-20T09:01:54,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290. as already flushing 2024-11-20T09:01:54,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290. 2024-11-20T09:01:54,672 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 2efe576d1bb7feeeb5d5727034c1a290, NAME => 'TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:54,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 2efe576d1bb7feeeb5d5727034c1a290, NAME => 'TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:54,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2efe576d1bb7feeeb5d5727034c1a290, NAME => 'TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2efe576d1bb7feeeb5d5727034c1a290, NAME => 'TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:54,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-20T09:01:54,824 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-20T09:01:54,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290. 2024-11-20T09:01:54,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290. as already flushing 2024-11-20T09:01:54,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290. 2024-11-20T09:01:54,825 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 2efe576d1bb7feeeb5d5727034c1a290, NAME => 'TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:54,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 2efe576d1bb7feeeb5d5727034c1a290, NAME => 'TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:54,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2efe576d1bb7feeeb5d5727034c1a290, NAME => 'TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2efe576d1bb7feeeb5d5727034c1a290, NAME => 'TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:54,880 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:54,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53418 deadline: 1732093374877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:54,880 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:54,880 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:54,880 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:54,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53442 deadline: 1732093374877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:54,880 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:54,880 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:54,881 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:54,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53446 deadline: 1732093374877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:54,881 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:54,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53372 deadline: 1732093374877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:54,881 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:54,881 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:54,881 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:54,881 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:54,881 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:54,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53462 deadline: 1732093374877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:54,881 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:54,881 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:54,881 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:54,881 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:54,881 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:54,881 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:54,881 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:54,977 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-20T09:01:54,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290. 2024-11-20T09:01:54,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290. as already flushing 2024-11-20T09:01:54,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290. 2024-11-20T09:01:54,978 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 2efe576d1bb7feeeb5d5727034c1a290, NAME => 'TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:54,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 2efe576d1bb7feeeb5d5727034c1a290, NAME => 'TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:54,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2efe576d1bb7feeeb5d5727034c1a290, NAME => 'TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2efe576d1bb7feeeb5d5727034c1a290, NAME => 'TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:55,015 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=53, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/.tmp/A/763115a23ee04e84a31acd8f405995e7 2024-11-20T09:01:55,022 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/.tmp/B/e1646054bd104f1c962cf971cdf08c3c is 50, key is test_row_0/B:col10/1732093314143/Put/seqid=0 2024-11-20T09:01:55,026 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742394_1570 (size=12001) 2024-11-20T09:01:55,131 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-20T09:01:55,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290. 2024-11-20T09:01:55,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290. as already flushing 2024-11-20T09:01:55,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290. 2024-11-20T09:01:55,131 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 2efe576d1bb7feeeb5d5727034c1a290, NAME => 'TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:55,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 2efe576d1bb7feeeb5d5727034c1a290, NAME => 'TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:55,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2efe576d1bb7feeeb5d5727034c1a290, NAME => 'TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2efe576d1bb7feeeb5d5727034c1a290, NAME => 'TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:55,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-20T09:01:55,283 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-20T09:01:55,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290. 2024-11-20T09:01:55,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290. as already flushing 2024-11-20T09:01:55,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290. 2024-11-20T09:01:55,284 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 2efe576d1bb7feeeb5d5727034c1a290, NAME => 'TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:55,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 2efe576d1bb7feeeb5d5727034c1a290, NAME => 'TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:55,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2efe576d1bb7feeeb5d5727034c1a290, NAME => 'TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2efe576d1bb7feeeb5d5727034c1a290, NAME => 'TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:55,389 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:55,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53418 deadline: 1732093375387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:55,390 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:55,390 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:55,390 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:55,390 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:55,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53442 deadline: 1732093375387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:55,390 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:55,390 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:55,390 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:55,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53372 deadline: 1732093375387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:55,390 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:55,390 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:55,390 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:55,390 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:55,390 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:55,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53446 deadline: 1732093375387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:55,391 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:55,391 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:55,391 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:55,391 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:55,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53462 deadline: 1732093375387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:55,391 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:55,391 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:55,391 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:55,426 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=53 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/.tmp/B/e1646054bd104f1c962cf971cdf08c3c 2024-11-20T09:01:55,432 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/.tmp/C/beba8a8f994d40daa025c3e35b40d25c is 50, key is test_row_0/C:col10/1732093314143/Put/seqid=0 2024-11-20T09:01:55,436 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742395_1571 (size=12001) 2024-11-20T09:01:55,436 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-20T09:01:55,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290. 2024-11-20T09:01:55,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290. as already flushing 2024-11-20T09:01:55,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290. 2024-11-20T09:01:55,437 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 2efe576d1bb7feeeb5d5727034c1a290, NAME => 'TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:55,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 2efe576d1bb7feeeb5d5727034c1a290, NAME => 'TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:55,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2efe576d1bb7feeeb5d5727034c1a290, NAME => 'TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2efe576d1bb7feeeb5d5727034c1a290, NAME => 'TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:55,589 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-20T09:01:55,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290. 2024-11-20T09:01:55,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290. as already flushing 2024-11-20T09:01:55,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290. 2024-11-20T09:01:55,589 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 2efe576d1bb7feeeb5d5727034c1a290, NAME => 'TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:55,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 2efe576d1bb7feeeb5d5727034c1a290, NAME => 'TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:55,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2efe576d1bb7feeeb5d5727034c1a290, NAME => 'TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2efe576d1bb7feeeb5d5727034c1a290, NAME => 'TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:55,742 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-20T09:01:55,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290. 2024-11-20T09:01:55,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290. as already flushing 2024-11-20T09:01:55,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290. 2024-11-20T09:01:55,742 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 2efe576d1bb7feeeb5d5727034c1a290, NAME => 'TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:55,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 2efe576d1bb7feeeb5d5727034c1a290, NAME => 'TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:55,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2efe576d1bb7feeeb5d5727034c1a290, NAME => 'TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2efe576d1bb7feeeb5d5727034c1a290, NAME => 'TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:01:55,836 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=53 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/.tmp/C/beba8a8f994d40daa025c3e35b40d25c 2024-11-20T09:01:55,840 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/.tmp/A/763115a23ee04e84a31acd8f405995e7 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/A/763115a23ee04e84a31acd8f405995e7 2024-11-20T09:01:55,843 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/A/763115a23ee04e84a31acd8f405995e7, entries=400, sequenceid=53, filesize=72.3 K 2024-11-20T09:01:55,844 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/.tmp/B/e1646054bd104f1c962cf971cdf08c3c as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/B/e1646054bd104f1c962cf971cdf08c3c 2024-11-20T09:01:55,847 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/B/e1646054bd104f1c962cf971cdf08c3c, entries=150, sequenceid=53, filesize=11.7 K 2024-11-20T09:01:55,847 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/.tmp/C/beba8a8f994d40daa025c3e35b40d25c as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/C/beba8a8f994d40daa025c3e35b40d25c 2024-11-20T09:01:55,850 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/C/beba8a8f994d40daa025c3e35b40d25c, entries=150, sequenceid=53, filesize=11.7 K 2024-11-20T09:01:55,851 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=154.31 KB/158010 for 2efe576d1bb7feeeb5d5727034c1a290 in 1676ms, sequenceid=53, compaction requested=true 2024-11-20T09:01:55,851 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2efe576d1bb7feeeb5d5727034c1a290: 2024-11-20T09:01:55,851 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2efe576d1bb7feeeb5d5727034c1a290:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T09:01:55,851 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T09:01:55,851 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2efe576d1bb7feeeb5d5727034c1a290:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T09:01:55,851 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T09:01:55,851 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2efe576d1bb7feeeb5d5727034c1a290:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T09:01:55,851 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T09:01:55,851 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T09:01:55,851 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T09:01:55,852 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 135904 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T09:01:55,852 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T09:01:55,852 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1541): 2efe576d1bb7feeeb5d5727034c1a290/B is initiating minor compaction (all files) 2024-11-20T09:01:55,852 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HStore(1541): 2efe576d1bb7feeeb5d5727034c1a290/A is initiating minor compaction (all files) 2024-11-20T09:01:55,852 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2efe576d1bb7feeeb5d5727034c1a290/B in TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290. 2024-11-20T09:01:55,852 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2efe576d1bb7feeeb5d5727034c1a290/A in TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290. 2024-11-20T09:01:55,852 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/B/919cbddd2375495abf2a7ae69c21d872, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/B/cdd4ea03fc7b4a1e8cd7e4d5d2a62046, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/B/e1646054bd104f1c962cf971cdf08c3c] into tmpdir=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/.tmp, totalSize=35.2 K 2024-11-20T09:01:55,852 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/A/204bd333446043fa8a8c9beb08a5f232, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/A/737b2ab810cd4f8d887134abda2f5634, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/A/763115a23ee04e84a31acd8f405995e7] into tmpdir=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/.tmp, totalSize=132.7 K 2024-11-20T09:01:55,852 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290. 2024-11-20T09:01:55,852 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290. files: [hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/A/204bd333446043fa8a8c9beb08a5f232, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/A/737b2ab810cd4f8d887134abda2f5634, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/A/763115a23ee04e84a31acd8f405995e7] 2024-11-20T09:01:55,852 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting 919cbddd2375495abf2a7ae69c21d872, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732093311920 2024-11-20T09:01:55,853 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting cdd4ea03fc7b4a1e8cd7e4d5d2a62046, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=42, earliestPutTs=1732093311951 2024-11-20T09:01:55,853 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.Compactor(225): Compacting 204bd333446043fa8a8c9beb08a5f232, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732093311920 2024-11-20T09:01:55,853 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.Compactor(225): Compacting 737b2ab810cd4f8d887134abda2f5634, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=42, earliestPutTs=1732093311951 2024-11-20T09:01:55,853 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting e1646054bd104f1c962cf971cdf08c3c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1732093314143 2024-11-20T09:01:55,853 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.Compactor(225): Compacting 763115a23ee04e84a31acd8f405995e7, keycount=400, bloomtype=ROW, size=72.3 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1732093314141 2024-11-20T09:01:55,862 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2efe576d1bb7feeeb5d5727034c1a290#B#compaction#488 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T09:01:55,863 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/.tmp/B/7e9e4b7ab89e4728af657370090f5bee is 50, key is test_row_0/B:col10/1732093314143/Put/seqid=0 2024-11-20T09:01:55,864 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=2efe576d1bb7feeeb5d5727034c1a290] 2024-11-20T09:01:55,865 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411205ca3941eb40445339fa9e8dc31073967_2efe576d1bb7feeeb5d5727034c1a290 store=[table=TestAcidGuarantees family=A region=2efe576d1bb7feeeb5d5727034c1a290] 2024-11-20T09:01:55,867 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411205ca3941eb40445339fa9e8dc31073967_2efe576d1bb7feeeb5d5727034c1a290, store=[table=TestAcidGuarantees family=A region=2efe576d1bb7feeeb5d5727034c1a290] 2024-11-20T09:01:55,867 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742396_1572 (size=12104) 2024-11-20T09:01:55,867 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411205ca3941eb40445339fa9e8dc31073967_2efe576d1bb7feeeb5d5727034c1a290 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=2efe576d1bb7feeeb5d5727034c1a290] 2024-11-20T09:01:55,871 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742397_1573 (size=4469) 2024-11-20T09:01:55,872 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2efe576d1bb7feeeb5d5727034c1a290#A#compaction#489 average throughput is 3.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T09:01:55,872 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/.tmp/A/7c9f0c17f70b4264bf90e44a3c33d8f8 is 175, key is test_row_0/A:col10/1732093314143/Put/seqid=0 2024-11-20T09:01:55,875 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742398_1574 (size=31058) 2024-11-20T09:01:55,879 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/.tmp/A/7c9f0c17f70b4264bf90e44a3c33d8f8 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/A/7c9f0c17f70b4264bf90e44a3c33d8f8 2024-11-20T09:01:55,884 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2efe576d1bb7feeeb5d5727034c1a290/A of 2efe576d1bb7feeeb5d5727034c1a290 into 7c9f0c17f70b4264bf90e44a3c33d8f8(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T09:01:55,884 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2efe576d1bb7feeeb5d5727034c1a290: 2024-11-20T09:01:55,884 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., storeName=2efe576d1bb7feeeb5d5727034c1a290/A, priority=13, startTime=1732093315851; duration=0sec 2024-11-20T09:01:55,884 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T09:01:55,884 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2efe576d1bb7feeeb5d5727034c1a290:A 2024-11-20T09:01:55,884 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T09:01:55,884 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T09:01:55,885 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HStore(1541): 2efe576d1bb7feeeb5d5727034c1a290/C is initiating minor compaction (all files) 2024-11-20T09:01:55,885 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2efe576d1bb7feeeb5d5727034c1a290/C in TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290. 2024-11-20T09:01:55,885 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/C/bfd37f541ac94c37a67737eca6a42199, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/C/22dac69b05d8410a9ca31ddc8782e016, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/C/beba8a8f994d40daa025c3e35b40d25c] into tmpdir=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/.tmp, totalSize=35.2 K 2024-11-20T09:01:55,885 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.Compactor(225): Compacting bfd37f541ac94c37a67737eca6a42199, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732093311920 2024-11-20T09:01:55,885 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.Compactor(225): Compacting 22dac69b05d8410a9ca31ddc8782e016, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=42, earliestPutTs=1732093311951 2024-11-20T09:01:55,885 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.Compactor(225): Compacting beba8a8f994d40daa025c3e35b40d25c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1732093314143 2024-11-20T09:01:55,891 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2efe576d1bb7feeeb5d5727034c1a290#C#compaction#490 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T09:01:55,891 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/.tmp/C/6a6388e84b7b47f38ebe2f81cec5a14f is 50, key is test_row_0/C:col10/1732093314143/Put/seqid=0 2024-11-20T09:01:55,895 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-20T09:01:55,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290. 2024-11-20T09:01:55,895 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2902): Flushing 2efe576d1bb7feeeb5d5727034c1a290 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-20T09:01:55,897 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742399_1575 (size=12104) 2024-11-20T09:01:55,904 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/.tmp/C/6a6388e84b7b47f38ebe2f81cec5a14f as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/C/6a6388e84b7b47f38ebe2f81cec5a14f 2024-11-20T09:01:55,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112011ff8a7cf7354cecb45b47d16fe50737_2efe576d1bb7feeeb5d5727034c1a290 is 50, key is test_row_0/A:col10/1732093314245/Put/seqid=0 2024-11-20T09:01:55,910 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2efe576d1bb7feeeb5d5727034c1a290/C of 2efe576d1bb7feeeb5d5727034c1a290 into 6a6388e84b7b47f38ebe2f81cec5a14f(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T09:01:55,911 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2efe576d1bb7feeeb5d5727034c1a290: 2024-11-20T09:01:55,911 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., storeName=2efe576d1bb7feeeb5d5727034c1a290/C, priority=13, startTime=1732093315851; duration=0sec 2024-11-20T09:01:55,911 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T09:01:55,911 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2efe576d1bb7feeeb5d5727034c1a290:C 2024-11-20T09:01:55,921 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742400_1576 (size=12154) 2024-11-20T09:01:56,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-20T09:01:56,272 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/.tmp/B/7e9e4b7ab89e4728af657370090f5bee as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/B/7e9e4b7ab89e4728af657370090f5bee 2024-11-20T09:01:56,277 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2efe576d1bb7feeeb5d5727034c1a290/B of 2efe576d1bb7feeeb5d5727034c1a290 into 7e9e4b7ab89e4728af657370090f5bee(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T09:01:56,277 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2efe576d1bb7feeeb5d5727034c1a290: 2024-11-20T09:01:56,277 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., storeName=2efe576d1bb7feeeb5d5727034c1a290/B, priority=13, startTime=1732093315851; duration=0sec 2024-11-20T09:01:56,277 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T09:01:56,277 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2efe576d1bb7feeeb5d5727034c1a290:B 2024-11-20T09:01:56,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:56,325 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112011ff8a7cf7354cecb45b47d16fe50737_2efe576d1bb7feeeb5d5727034c1a290 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112011ff8a7cf7354cecb45b47d16fe50737_2efe576d1bb7feeeb5d5727034c1a290 2024-11-20T09:01:56,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/.tmp/A/88a6fdd2c5f14f78a80a75e3bffb79f8, store: [table=TestAcidGuarantees family=A region=2efe576d1bb7feeeb5d5727034c1a290] 2024-11-20T09:01:56,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/.tmp/A/88a6fdd2c5f14f78a80a75e3bffb79f8 is 175, key is test_row_0/A:col10/1732093314245/Put/seqid=0 2024-11-20T09:01:56,330 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742401_1577 (size=30955) 2024-11-20T09:01:56,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(8855): Flush requested on 2efe576d1bb7feeeb5d5727034c1a290 2024-11-20T09:01:56,400 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290. as already flushing 2024-11-20T09:01:56,410 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:56,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53442 deadline: 1732093376405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:56,410 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:56,410 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:56,410 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:56,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53372 deadline: 1732093376406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:56,410 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:56,411 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:56,411 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:56,411 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:56,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53418 deadline: 1732093376406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:56,411 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:56,411 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:56,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53446 deadline: 1732093376407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:56,411 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:56,411 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:56,411 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:56,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53462 deadline: 1732093376407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:56,411 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:56,411 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to be1eb2620e0e,34223,1732093159960 for 'test_row_1' in 2efe576d1bb7feeeb5d5727034c1a290 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2167 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-20T09:01:56,412 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:56,412 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:56,412 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:56,412 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:56,412 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:56,412 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:56,519 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:56,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53442 deadline: 1732093376516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:56,519 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:56,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53372 deadline: 1732093376517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:56,519 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:56,519 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:56,519 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:56,519 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:56,520 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:56,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53446 deadline: 1732093376517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:56,520 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:56,520 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:56,520 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:56,520 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:56,520 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:56,520 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:56,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53462 deadline: 1732093376517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:56,520 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:56,520 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:56,520 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:56,730 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:56,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53442 deadline: 1732093376727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:56,730 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:56,730 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:56,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53372 deadline: 1732093376727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:56,730 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:56,730 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:56,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53446 deadline: 1732093376727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:56,730 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:56,731 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=80, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/.tmp/A/88a6fdd2c5f14f78a80a75e3bffb79f8 2024-11-20T09:01:56,731 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:56,731 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:56,731 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:56,731 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:56,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53462 deadline: 1732093376727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:56,731 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:56,731 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:56,731 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:56,731 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:56,731 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:56,731 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:56,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/.tmp/B/8c9f66c0eef347d4b0fb682f5b5a90ac is 50, key is test_row_0/B:col10/1732093314245/Put/seqid=0 2024-11-20T09:01:56,740 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742402_1578 (size=12001) 2024-11-20T09:01:57,039 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:57,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53442 deadline: 1732093377037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:57,040 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:57,040 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:57,040 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:57,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53372 deadline: 1732093377037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:57,040 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:57,040 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:57,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53446 deadline: 1732093377037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:57,040 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:57,040 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:57,040 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:57,040 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:57,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53462 deadline: 1732093377037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:57,040 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:57,040 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:57,040 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:57,040 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:57,040 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:57,040 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:57,141 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=80 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/.tmp/B/8c9f66c0eef347d4b0fb682f5b5a90ac 2024-11-20T09:01:57,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/.tmp/C/6aadc71ddde848bd85515c14c179107b is 50, key is test_row_0/C:col10/1732093314245/Put/seqid=0 2024-11-20T09:01:57,151 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742403_1579 (size=12001) 2024-11-20T09:01:57,549 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:57,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53442 deadline: 1732093377547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:57,550 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:57,550 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:57,550 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:57,550 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:57,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53372 deadline: 1732093377547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:57,550 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:57,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53446 deadline: 1732093377547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:57,551 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:57,551 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:57,551 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:57,551 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:57,551 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:57,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53462 deadline: 1732093377547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:57,551 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:57,551 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:57,551 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:57,551 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:57,551 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:57,552 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=80 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/.tmp/C/6aadc71ddde848bd85515c14c179107b 2024-11-20T09:01:57,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/.tmp/A/88a6fdd2c5f14f78a80a75e3bffb79f8 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/A/88a6fdd2c5f14f78a80a75e3bffb79f8 2024-11-20T09:01:57,559 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/A/88a6fdd2c5f14f78a80a75e3bffb79f8, entries=150, sequenceid=80, filesize=30.2 K 2024-11-20T09:01:57,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/.tmp/B/8c9f66c0eef347d4b0fb682f5b5a90ac as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/B/8c9f66c0eef347d4b0fb682f5b5a90ac 2024-11-20T09:01:57,562 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/B/8c9f66c0eef347d4b0fb682f5b5a90ac, entries=150, sequenceid=80, filesize=11.7 K 2024-11-20T09:01:57,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/.tmp/C/6aadc71ddde848bd85515c14c179107b as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/C/6aadc71ddde848bd85515c14c179107b 2024-11-20T09:01:57,566 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/C/6aadc71ddde848bd85515c14c179107b, entries=150, sequenceid=80, filesize=11.7 K 2024-11-20T09:01:57,567 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=53.67 KB/54960 for 2efe576d1bb7feeeb5d5727034c1a290 in 1672ms, sequenceid=80, compaction requested=false 2024-11-20T09:01:57,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2603): Flush status journal for 2efe576d1bb7feeeb5d5727034c1a290: 2024-11-20T09:01:57,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290. 2024-11-20T09:01:57,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=147 2024-11-20T09:01:57,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.HMaster(4169): Remote procedure done, pid=147 2024-11-20T09:01:57,570 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=147, resume processing ppid=146 2024-11-20T09:01:57,570 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=147, ppid=146, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.5070 sec 2024-11-20T09:01:57,572 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=146, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees in 3.5130 sec 2024-11-20T09:01:58,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-20T09:01:58,217 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T09:01:58,217 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T09:01:58,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] procedure2.ProcedureExecutor(1139): Stored pid=148, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees 2024-11-20T09:01:58,219 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=148, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T09:01:58,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-20T09:01:58,220 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=148, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T09:01:58,220 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=149, ppid=148, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T09:01:58,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-20T09:01:58,373 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-20T09:01:58,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290. 2024-11-20T09:01:58,373 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2902): Flushing 2efe576d1bb7feeeb5d5727034c1a290 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-20T09:01:58,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120a1d04945caa34377868db9913d604cce_2efe576d1bb7feeeb5d5727034c1a290 is 50, key is test_row_0/A:col10/1732093316400/Put/seqid=0 2024-11-20T09:01:58,383 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742404_1580 (size=12154) 2024-11-20T09:01:58,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(8855): Flush requested on 2efe576d1bb7feeeb5d5727034c1a290 2024-11-20T09:01:58,429 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290. as already flushing 2024-11-20T09:01:58,526 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:58,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53418 deadline: 1732093378522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:58,526 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:58,526 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:58,526 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:58,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-20T09:01:58,558 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:58,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53442 deadline: 1732093378556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:58,559 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:58,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53372 deadline: 1732093378556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:58,559 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:58,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53446 deadline: 1732093378557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:58,559 WARN [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRpcRetryingCaller(177): Call to be1eb2620e0e,34223,1732093159960 for 'test_row_1' in 2efe576d1bb7feeeb5d5727034c1a290 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2153 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-20T09:01:58,559 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:58,559 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:58,559 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to be1eb2620e0e,34223,1732093159960 for 'test_row_0' in 2efe576d1bb7feeeb5d5727034c1a290 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2153 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-20T09:01:58,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53462 deadline: 1732093378557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:58,560 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:58,560 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:58,560 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:58,560 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:58,560 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:58,559 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to be1eb2620e0e,34223,1732093159960 for 'test_row_1' in 2efe576d1bb7feeeb5d5727034c1a290 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2153 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-20T09:01:58,560 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:58,560 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:58,560 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:58,560 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to be1eb2620e0e,34223,1732093159960 for 'test_row_1' in 2efe576d1bb7feeeb5d5727034c1a290 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2153 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-20T09:01:58,560 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:58,561 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:58,561 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:58,639 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:58,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53418 deadline: 1732093378636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:58,640 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:58,640 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:58,640 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:58,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:58,788 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120a1d04945caa34377868db9913d604cce_2efe576d1bb7feeeb5d5727034c1a290 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120a1d04945caa34377868db9913d604cce_2efe576d1bb7feeeb5d5727034c1a290 2024-11-20T09:01:58,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/.tmp/A/155c144420354405bee0048adb1006dc, store: [table=TestAcidGuarantees family=A region=2efe576d1bb7feeeb5d5727034c1a290] 2024-11-20T09:01:58,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/.tmp/A/155c144420354405bee0048adb1006dc is 175, key is test_row_0/A:col10/1732093316400/Put/seqid=0 2024-11-20T09:01:58,797 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742405_1581 (size=30955) 2024-11-20T09:01:58,797 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=93, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/.tmp/A/155c144420354405bee0048adb1006dc 2024-11-20T09:01:58,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/.tmp/B/188e0c11698f458a86aa3d4de45b0f31 is 50, key is test_row_0/B:col10/1732093316400/Put/seqid=0 2024-11-20T09:01:58,806 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742406_1582 (size=12001) 2024-11-20T09:01:58,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-20T09:01:58,849 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:58,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53418 deadline: 1732093378847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:58,850 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:58,850 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:58,850 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:59,159 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:59,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53418 deadline: 1732093379157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:59,159 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:59,159 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:59,159 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:59,207 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/.tmp/B/188e0c11698f458a86aa3d4de45b0f31 2024-11-20T09:01:59,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/.tmp/C/848c0c5a29ad4244ac07408b9cf158f7 is 50, key is test_row_0/C:col10/1732093316400/Put/seqid=0 2024-11-20T09:01:59,218 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742407_1583 (size=12001) 2024-11-20T09:01:59,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-20T09:01:59,618 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/.tmp/C/848c0c5a29ad4244ac07408b9cf158f7 2024-11-20T09:01:59,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/.tmp/A/155c144420354405bee0048adb1006dc as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/A/155c144420354405bee0048adb1006dc 2024-11-20T09:01:59,625 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/A/155c144420354405bee0048adb1006dc, entries=150, sequenceid=93, filesize=30.2 K 2024-11-20T09:01:59,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/.tmp/B/188e0c11698f458a86aa3d4de45b0f31 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/B/188e0c11698f458a86aa3d4de45b0f31 2024-11-20T09:01:59,629 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/B/188e0c11698f458a86aa3d4de45b0f31, entries=150, sequenceid=93, filesize=11.7 K 2024-11-20T09:01:59,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/.tmp/C/848c0c5a29ad4244ac07408b9cf158f7 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/C/848c0c5a29ad4244ac07408b9cf158f7 2024-11-20T09:01:59,633 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/C/848c0c5a29ad4244ac07408b9cf158f7, entries=150, sequenceid=93, filesize=11.7 K 2024-11-20T09:01:59,634 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 2efe576d1bb7feeeb5d5727034c1a290 in 1260ms, sequenceid=93, compaction requested=true 2024-11-20T09:01:59,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2603): Flush status journal for 2efe576d1bb7feeeb5d5727034c1a290: 2024-11-20T09:01:59,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290. 2024-11-20T09:01:59,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=149 2024-11-20T09:01:59,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.HMaster(4169): Remote procedure done, pid=149 2024-11-20T09:01:59,637 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=149, resume processing ppid=148 2024-11-20T09:01:59,637 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=149, ppid=148, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4150 sec 2024-11-20T09:01:59,638 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=148, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees in 1.4200 sec 2024-11-20T09:01:59,645 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-20T09:01:59,645 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-20T09:01:59,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(8855): Flush requested on 2efe576d1bb7feeeb5d5727034c1a290 2024-11-20T09:01:59,669 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T09:01:59,670 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2efe576d1bb7feeeb5d5727034c1a290 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-20T09:01:59,677 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112015650b08cc1d4a65a407beb964f58fae_2efe576d1bb7feeeb5d5727034c1a290 is 50, key is test_row_0/A:col10/1732093319668/Put/seqid=0 2024-11-20T09:01:59,680 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742408_1584 (size=12154) 2024-11-20T09:01:59,681 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:01:59,684 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112015650b08cc1d4a65a407beb964f58fae_2efe576d1bb7feeeb5d5727034c1a290 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112015650b08cc1d4a65a407beb964f58fae_2efe576d1bb7feeeb5d5727034c1a290 2024-11-20T09:01:59,685 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/.tmp/A/45b817b98eec4a1c85086a07ac8101e9, store: [table=TestAcidGuarantees family=A region=2efe576d1bb7feeeb5d5727034c1a290] 2024-11-20T09:01:59,685 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/.tmp/A/45b817b98eec4a1c85086a07ac8101e9 is 175, key is test_row_0/A:col10/1732093319668/Put/seqid=0 2024-11-20T09:01:59,703 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:59,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53418 deadline: 1732093379700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:59,703 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:59,703 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:59,703 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:01:59,704 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742409_1585 (size=30955) 2024-11-20T09:01:59,809 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:01:59,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53418 deadline: 1732093379807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:01:59,810 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:59,810 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:01:59,810 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:02:00,019 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:02:00,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53418 deadline: 1732093380016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:02:00,020 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:02:00,020 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:02:00,020 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:02:00,105 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=119, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/.tmp/A/45b817b98eec4a1c85086a07ac8101e9 2024-11-20T09:02:00,112 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/.tmp/B/ace8ca1ee71547e8b283eee4a8b7e6a0 is 50, key is test_row_0/B:col10/1732093319668/Put/seqid=0 2024-11-20T09:02:00,115 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742410_1586 (size=12001) 2024-11-20T09:02:00,330 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:02:00,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53418 deadline: 1732093380327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:02:00,331 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:02:00,331 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:02:00,331 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:02:00,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-20T09:02:00,368 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T09:02:00,368 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T09:02:00,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] procedure2.ProcedureExecutor(1139): Stored pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees 2024-11-20T09:02:00,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-20T09:02:00,370 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T09:02:00,371 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T09:02:00,371 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=151, ppid=150, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T09:02:00,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-20T09:02:00,516 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/.tmp/B/ace8ca1ee71547e8b283eee4a8b7e6a0 2024-11-20T09:02:00,522 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/.tmp/C/f9c80e049fe34754bc930e8430edc6c9 is 50, key is test_row_0/C:col10/1732093319668/Put/seqid=0 2024-11-20T09:02:00,523 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-20T09:02:00,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290. 2024-11-20T09:02:00,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290. as already flushing 2024-11-20T09:02:00,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290. 2024-11-20T09:02:00,524 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 2efe576d1bb7feeeb5d5727034c1a290, NAME => 'TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:02:00,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 2efe576d1bb7feeeb5d5727034c1a290, NAME => 'TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:02:00,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2efe576d1bb7feeeb5d5727034c1a290, NAME => 'TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2efe576d1bb7feeeb5d5727034c1a290, NAME => 'TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:02:00,527 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742411_1587 (size=12001) 2024-11-20T09:02:00,570 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:02:00,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53442 deadline: 1732093380567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:02:00,570 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:02:00,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53372 deadline: 1732093380567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:02:00,570 WARN [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRpcRetryingCaller(177): Call to be1eb2620e0e,34223,1732093159960 for 'test_row_1' in 2efe576d1bb7feeeb5d5727034c1a290 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4165 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-20T09:02:00,571 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to be1eb2620e0e,34223,1732093159960 for 'test_row_0' in 2efe576d1bb7feeeb5d5727034c1a290 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4165 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-20T09:02:00,571 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:02:00,571 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:02:00,571 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:02:00,571 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:02:00,571 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:02:00,571 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:02:00,580 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:02:00,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53446 deadline: 1732093380577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:02:00,580 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:02:00,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53462 deadline: 1732093380577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:02:00,580 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to be1eb2620e0e,34223,1732093159960 for 'test_row_1' in 2efe576d1bb7feeeb5d5727034c1a290 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4173 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-20T09:02:00,580 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to be1eb2620e0e,34223,1732093159960 for 'test_row_1' in 2efe576d1bb7feeeb5d5727034c1a290 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4173 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-20T09:02:00,581 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:02:00,581 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:02:00,581 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:02:00,581 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:02:00,581 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:02:00,581 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:02:00,676 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-20T09:02:00,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290. 2024-11-20T09:02:00,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290. as already flushing 2024-11-20T09:02:00,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290. 2024-11-20T09:02:00,677 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 2efe576d1bb7feeeb5d5727034c1a290, NAME => 'TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:02:00,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 2efe576d1bb7feeeb5d5727034c1a290, NAME => 'TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:02:00,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2efe576d1bb7feeeb5d5727034c1a290, NAME => 'TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2efe576d1bb7feeeb5d5727034c1a290, NAME => 'TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:02:00,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-20T09:02:00,829 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-20T09:02:00,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290. 2024-11-20T09:02:00,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290. as already flushing 2024-11-20T09:02:00,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290. 2024-11-20T09:02:00,830 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 2efe576d1bb7feeeb5d5727034c1a290, NAME => 'TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:02:00,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 2efe576d1bb7feeeb5d5727034c1a290, NAME => 'TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:02:00,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2efe576d1bb7feeeb5d5727034c1a290, NAME => 'TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2efe576d1bb7feeeb5d5727034c1a290, NAME => 'TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:02:00,841 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:02:00,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53418 deadline: 1732093380837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:02:00,842 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:02:00,842 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:02:00,842 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:02:00,927 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/.tmp/C/f9c80e049fe34754bc930e8430edc6c9 2024-11-20T09:02:00,931 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/.tmp/A/45b817b98eec4a1c85086a07ac8101e9 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/A/45b817b98eec4a1c85086a07ac8101e9 2024-11-20T09:02:00,934 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/A/45b817b98eec4a1c85086a07ac8101e9, entries=150, sequenceid=119, filesize=30.2 K 2024-11-20T09:02:00,935 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/.tmp/B/ace8ca1ee71547e8b283eee4a8b7e6a0 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/B/ace8ca1ee71547e8b283eee4a8b7e6a0 2024-11-20T09:02:00,938 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/B/ace8ca1ee71547e8b283eee4a8b7e6a0, entries=150, sequenceid=119, filesize=11.7 K 2024-11-20T09:02:00,938 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/.tmp/C/f9c80e049fe34754bc930e8430edc6c9 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/C/f9c80e049fe34754bc930e8430edc6c9 2024-11-20T09:02:00,941 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/C/f9c80e049fe34754bc930e8430edc6c9, entries=150, sequenceid=119, filesize=11.7 K 2024-11-20T09:02:00,942 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 2efe576d1bb7feeeb5d5727034c1a290 in 1273ms, sequenceid=119, compaction requested=true 2024-11-20T09:02:00,942 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2efe576d1bb7feeeb5d5727034c1a290: 2024-11-20T09:02:00,942 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2efe576d1bb7feeeb5d5727034c1a290:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T09:02:00,942 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T09:02:00,942 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T09:02:00,942 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2efe576d1bb7feeeb5d5727034c1a290:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T09:02:00,942 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T09:02:00,942 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2efe576d1bb7feeeb5d5727034c1a290:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T09:02:00,942 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T09:02:00,942 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T09:02:00,943 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 123923 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T09:02:00,943 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T09:02:00,944 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1541): 2efe576d1bb7feeeb5d5727034c1a290/B is initiating minor compaction (all files) 2024-11-20T09:02:00,944 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HStore(1541): 2efe576d1bb7feeeb5d5727034c1a290/A is initiating minor compaction (all files) 2024-11-20T09:02:00,944 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2efe576d1bb7feeeb5d5727034c1a290/A in TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290. 2024-11-20T09:02:00,944 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2efe576d1bb7feeeb5d5727034c1a290/B in TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290. 2024-11-20T09:02:00,944 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/A/7c9f0c17f70b4264bf90e44a3c33d8f8, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/A/88a6fdd2c5f14f78a80a75e3bffb79f8, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/A/155c144420354405bee0048adb1006dc, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/A/45b817b98eec4a1c85086a07ac8101e9] into tmpdir=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/.tmp, totalSize=121.0 K 2024-11-20T09:02:00,944 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/B/7e9e4b7ab89e4728af657370090f5bee, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/B/8c9f66c0eef347d4b0fb682f5b5a90ac, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/B/188e0c11698f458a86aa3d4de45b0f31, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/B/ace8ca1ee71547e8b283eee4a8b7e6a0] into tmpdir=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/.tmp, totalSize=47.0 K 2024-11-20T09:02:00,944 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290. 2024-11-20T09:02:00,944 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290. files: [hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/A/7c9f0c17f70b4264bf90e44a3c33d8f8, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/A/88a6fdd2c5f14f78a80a75e3bffb79f8, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/A/155c144420354405bee0048adb1006dc, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/A/45b817b98eec4a1c85086a07ac8101e9] 2024-11-20T09:02:00,944 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting 7e9e4b7ab89e4728af657370090f5bee, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1732093314143 2024-11-20T09:02:00,944 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7c9f0c17f70b4264bf90e44a3c33d8f8, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1732093314143 2024-11-20T09:02:00,944 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.Compactor(225): Compacting 88a6fdd2c5f14f78a80a75e3bffb79f8, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1732093314236 2024-11-20T09:02:00,944 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting 8c9f66c0eef347d4b0fb682f5b5a90ac, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1732093314236 2024-11-20T09:02:00,945 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.Compactor(225): Compacting 155c144420354405bee0048adb1006dc, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1732093316400 2024-11-20T09:02:00,945 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting 188e0c11698f458a86aa3d4de45b0f31, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1732093316400 2024-11-20T09:02:00,945 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.Compactor(225): Compacting 45b817b98eec4a1c85086a07ac8101e9, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1732093318510 2024-11-20T09:02:00,945 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting ace8ca1ee71547e8b283eee4a8b7e6a0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1732093318510 2024-11-20T09:02:00,951 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=2efe576d1bb7feeeb5d5727034c1a290] 2024-11-20T09:02:00,953 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2efe576d1bb7feeeb5d5727034c1a290#B#compaction#501 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-20T09:02:00,954 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/.tmp/B/9ca1284dbf6842f1a7001bfe7ba44eeb is 50, key is test_row_0/B:col10/1732093319668/Put/seqid=0 2024-11-20T09:02:00,955 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411203faf440ef5404451b714bd8eb713a764_2efe576d1bb7feeeb5d5727034c1a290 store=[table=TestAcidGuarantees family=A region=2efe576d1bb7feeeb5d5727034c1a290] 2024-11-20T09:02:00,956 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411203faf440ef5404451b714bd8eb713a764_2efe576d1bb7feeeb5d5727034c1a290, store=[table=TestAcidGuarantees family=A region=2efe576d1bb7feeeb5d5727034c1a290] 2024-11-20T09:02:00,956 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411203faf440ef5404451b714bd8eb713a764_2efe576d1bb7feeeb5d5727034c1a290 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=2efe576d1bb7feeeb5d5727034c1a290] 2024-11-20T09:02:00,960 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742413_1589 (size=4469) 2024-11-20T09:02:00,960 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742412_1588 (size=12241) 2024-11-20T09:02:00,964 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2efe576d1bb7feeeb5d5727034c1a290#A#compaction#500 average throughput is 1.88 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T09:02:00,965 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/.tmp/A/f288fe87ca7141bf99732dfa7941b872 is 175, key is test_row_0/A:col10/1732093319668/Put/seqid=0 2024-11-20T09:02:00,968 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/.tmp/B/9ca1284dbf6842f1a7001bfe7ba44eeb as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/B/9ca1284dbf6842f1a7001bfe7ba44eeb 2024-11-20T09:02:00,969 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742414_1590 (size=31195) 2024-11-20T09:02:00,973 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 2efe576d1bb7feeeb5d5727034c1a290/B of 2efe576d1bb7feeeb5d5727034c1a290 into 9ca1284dbf6842f1a7001bfe7ba44eeb(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T09:02:00,973 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2efe576d1bb7feeeb5d5727034c1a290: 2024-11-20T09:02:00,973 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., storeName=2efe576d1bb7feeeb5d5727034c1a290/B, priority=12, startTime=1732093320942; duration=0sec 2024-11-20T09:02:00,973 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T09:02:00,973 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2efe576d1bb7feeeb5d5727034c1a290:B 2024-11-20T09:02:00,973 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T09:02:00,975 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T09:02:00,975 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1541): 2efe576d1bb7feeeb5d5727034c1a290/C is initiating minor compaction (all files) 2024-11-20T09:02:00,975 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2efe576d1bb7feeeb5d5727034c1a290/C in TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290. 2024-11-20T09:02:00,975 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/C/6a6388e84b7b47f38ebe2f81cec5a14f, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/C/6aadc71ddde848bd85515c14c179107b, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/C/848c0c5a29ad4244ac07408b9cf158f7, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/C/f9c80e049fe34754bc930e8430edc6c9] into tmpdir=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/.tmp, totalSize=47.0 K 2024-11-20T09:02:00,975 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting 6a6388e84b7b47f38ebe2f81cec5a14f, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1732093314143 2024-11-20T09:02:00,976 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting 6aadc71ddde848bd85515c14c179107b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1732093314236 2024-11-20T09:02:00,976 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting 848c0c5a29ad4244ac07408b9cf158f7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1732093316400 2024-11-20T09:02:00,976 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting f9c80e049fe34754bc930e8430edc6c9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1732093318510 2024-11-20T09:02:00,982 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-20T09:02:00,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290. 2024-11-20T09:02:00,983 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2902): Flushing 2efe576d1bb7feeeb5d5727034c1a290 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-20T09:02:00,984 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2efe576d1bb7feeeb5d5727034c1a290#C#compaction#502 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T09:02:00,985 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/.tmp/C/f118f2f791014323ba76947729c2bf22 is 50, key is test_row_0/C:col10/1732093319668/Put/seqid=0 2024-11-20T09:02:00,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411208c2f95454851462b8733de46c109417a_2efe576d1bb7feeeb5d5727034c1a290 is 50, key is test_row_0/A:col10/1732093319693/Put/seqid=0 2024-11-20T09:02:00,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-20T09:02:01,001 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742415_1591 (size=12241) 2024-11-20T09:02:01,006 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/.tmp/C/f118f2f791014323ba76947729c2bf22 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/C/f118f2f791014323ba76947729c2bf22 2024-11-20T09:02:01,010 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 2efe576d1bb7feeeb5d5727034c1a290/C of 2efe576d1bb7feeeb5d5727034c1a290 into f118f2f791014323ba76947729c2bf22(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T09:02:01,010 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2efe576d1bb7feeeb5d5727034c1a290: 2024-11-20T09:02:01,010 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., storeName=2efe576d1bb7feeeb5d5727034c1a290/C, priority=12, startTime=1732093320942; duration=0sec 2024-11-20T09:02:01,010 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T09:02:01,010 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2efe576d1bb7feeeb5d5727034c1a290:C 2024-11-20T09:02:01,013 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742416_1592 (size=12154) 2024-11-20T09:02:01,373 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/.tmp/A/f288fe87ca7141bf99732dfa7941b872 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/A/f288fe87ca7141bf99732dfa7941b872 2024-11-20T09:02:01,378 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 2efe576d1bb7feeeb5d5727034c1a290/A of 2efe576d1bb7feeeb5d5727034c1a290 into f288fe87ca7141bf99732dfa7941b872(size=30.5 K), total size for store is 30.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T09:02:01,378 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2efe576d1bb7feeeb5d5727034c1a290: 2024-11-20T09:02:01,378 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., storeName=2efe576d1bb7feeeb5d5727034c1a290/A, priority=12, startTime=1732093320942; duration=0sec 2024-11-20T09:02:01,378 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T09:02:01,378 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2efe576d1bb7feeeb5d5727034c1a290:A 2024-11-20T09:02:01,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:01,417 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411208c2f95454851462b8733de46c109417a_2efe576d1bb7feeeb5d5727034c1a290 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411208c2f95454851462b8733de46c109417a_2efe576d1bb7feeeb5d5727034c1a290 2024-11-20T09:02:01,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/.tmp/A/056d297ddd93429fbc52fefff1e2fee0, store: [table=TestAcidGuarantees family=A region=2efe576d1bb7feeeb5d5727034c1a290] 2024-11-20T09:02:01,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/.tmp/A/056d297ddd93429fbc52fefff1e2fee0 is 175, key is test_row_0/A:col10/1732093319693/Put/seqid=0 2024-11-20T09:02:01,422 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742417_1593 (size=30955) 2024-11-20T09:02:01,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-20T09:02:01,823 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=130, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/.tmp/A/056d297ddd93429fbc52fefff1e2fee0 2024-11-20T09:02:01,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/.tmp/B/544630a02cdf4b11a238078ccee09bf1 is 50, key is test_row_0/B:col10/1732093319693/Put/seqid=0 2024-11-20T09:02:01,840 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742418_1594 (size=12001) 2024-11-20T09:02:01,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(8855): Flush requested on 2efe576d1bb7feeeb5d5727034c1a290 2024-11-20T09:02:01,849 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290. as already flushing 2024-11-20T09:02:01,961 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:02:01,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53418 deadline: 1732093381957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:02:01,962 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:02:01,962 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:02:01,962 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:02:02,069 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:02:02,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53418 deadline: 1732093382066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:02:02,070 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:02:02,070 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:02:02,070 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:02:02,241 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/.tmp/B/544630a02cdf4b11a238078ccee09bf1 2024-11-20T09:02:02,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/.tmp/C/4df61a541594426a8a3d5dbe15cf7647 is 50, key is test_row_0/C:col10/1732093319693/Put/seqid=0 2024-11-20T09:02:02,250 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742419_1595 (size=12001) 2024-11-20T09:02:02,251 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/.tmp/C/4df61a541594426a8a3d5dbe15cf7647 2024-11-20T09:02:02,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/.tmp/A/056d297ddd93429fbc52fefff1e2fee0 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/A/056d297ddd93429fbc52fefff1e2fee0 2024-11-20T09:02:02,258 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/A/056d297ddd93429fbc52fefff1e2fee0, entries=150, sequenceid=130, filesize=30.2 K 2024-11-20T09:02:02,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/.tmp/B/544630a02cdf4b11a238078ccee09bf1 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/B/544630a02cdf4b11a238078ccee09bf1 2024-11-20T09:02:02,262 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/B/544630a02cdf4b11a238078ccee09bf1, entries=150, sequenceid=130, filesize=11.7 K 2024-11-20T09:02:02,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/.tmp/C/4df61a541594426a8a3d5dbe15cf7647 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/C/4df61a541594426a8a3d5dbe15cf7647 2024-11-20T09:02:02,265 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/C/4df61a541594426a8a3d5dbe15cf7647, entries=150, sequenceid=130, filesize=11.7 K 2024-11-20T09:02:02,266 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=154.31 KB/158010 for 2efe576d1bb7feeeb5d5727034c1a290 in 1284ms, sequenceid=130, compaction requested=false 2024-11-20T09:02:02,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2603): Flush status journal for 2efe576d1bb7feeeb5d5727034c1a290: 2024-11-20T09:02:02,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290. 2024-11-20T09:02:02,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=151 2024-11-20T09:02:02,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.HMaster(4169): Remote procedure done, pid=151 2024-11-20T09:02:02,269 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=151, resume processing ppid=150 2024-11-20T09:02:02,269 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=151, ppid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8970 sec 2024-11-20T09:02:02,271 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees in 1.9020 sec 2024-11-20T09:02:02,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(8855): Flush requested on 2efe576d1bb7feeeb5d5727034c1a290 2024-11-20T09:02:02,281 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T09:02:02,281 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2efe576d1bb7feeeb5d5727034c1a290 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-20T09:02:02,287 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120db9e3dc4e1e747f3bd1ab09cab660c29_2efe576d1bb7feeeb5d5727034c1a290 is 50, key is test_row_0/A:col10/1732093322280/Put/seqid=0 2024-11-20T09:02:02,291 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742420_1596 (size=14794) 2024-11-20T09:02:02,296 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:02,299 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120db9e3dc4e1e747f3bd1ab09cab660c29_2efe576d1bb7feeeb5d5727034c1a290 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120db9e3dc4e1e747f3bd1ab09cab660c29_2efe576d1bb7feeeb5d5727034c1a290 2024-11-20T09:02:02,300 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/.tmp/A/39631abd030a4b0389e9a5a04a1234e6, store: [table=TestAcidGuarantees family=A region=2efe576d1bb7feeeb5d5727034c1a290] 2024-11-20T09:02:02,301 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/.tmp/A/39631abd030a4b0389e9a5a04a1234e6 is 175, key is test_row_0/A:col10/1732093322280/Put/seqid=0 2024-11-20T09:02:02,304 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742421_1597 (size=39749) 2024-11-20T09:02:02,308 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:02:02,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53418 deadline: 1732093382306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:02:02,309 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:02:02,309 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:02:02,309 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:02:02,419 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:02:02,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53418 deadline: 1732093382417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:02:02,419 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:02:02,419 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:02:02,419 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:02:02,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-20T09:02:02,517 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T09:02:02,518 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T09:02:02,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] procedure2.ProcedureExecutor(1139): Stored pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees 2024-11-20T09:02:02,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-20T09:02:02,520 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T09:02:02,520 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T09:02:02,521 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=153, ppid=152, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T09:02:02,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-20T09:02:02,629 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:02:02,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53418 deadline: 1732093382627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:02:02,630 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:02:02,630 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:02:02,630 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:02:02,673 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-20T09:02:02,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290. 2024-11-20T09:02:02,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290. as already flushing 2024-11-20T09:02:02,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290. 2024-11-20T09:02:02,673 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 2efe576d1bb7feeeb5d5727034c1a290, NAME => 'TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:02:02,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 2efe576d1bb7feeeb5d5727034c1a290, NAME => 'TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:02:02,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2efe576d1bb7feeeb5d5727034c1a290, NAME => 'TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2efe576d1bb7feeeb5d5727034c1a290, NAME => 'TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:02:02,705 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=159, memsize=53.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/.tmp/A/39631abd030a4b0389e9a5a04a1234e6 2024-11-20T09:02:02,712 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/.tmp/B/ff0108518d3f4698a41614145284afb9 is 50, key is test_row_0/B:col10/1732093322280/Put/seqid=0 2024-11-20T09:02:02,720 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742422_1598 (size=12151) 2024-11-20T09:02:02,826 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-20T09:02:02,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290. 2024-11-20T09:02:02,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290. as already flushing 2024-11-20T09:02:02,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290. 2024-11-20T09:02:02,826 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 2efe576d1bb7feeeb5d5727034c1a290, NAME => 'TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:02:02,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 2efe576d1bb7feeeb5d5727034c1a290, NAME => 'TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:02:02,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2efe576d1bb7feeeb5d5727034c1a290, NAME => 'TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2efe576d1bb7feeeb5d5727034c1a290, NAME => 'TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:02:02,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-20T09:02:02,938 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:02:02,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53418 deadline: 1732093382937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:02:02,938 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:02:02,938 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:02:02,938 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:02:02,979 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-20T09:02:02,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290. 2024-11-20T09:02:02,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290. as already flushing 2024-11-20T09:02:02,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290. 2024-11-20T09:02:02,979 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 2efe576d1bb7feeeb5d5727034c1a290, NAME => 'TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:02:02,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 2efe576d1bb7feeeb5d5727034c1a290, NAME => 'TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:02:02,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2efe576d1bb7feeeb5d5727034c1a290, NAME => 'TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2efe576d1bb7feeeb5d5727034c1a290, NAME => 'TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:02:03,121 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=159 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/.tmp/B/ff0108518d3f4698a41614145284afb9 2024-11-20T09:02:03,129 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/.tmp/C/054ab2d467da47b48610e6b11130a4d4 is 50, key is test_row_0/C:col10/1732093322280/Put/seqid=0 2024-11-20T09:02:03,132 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-20T09:02:03,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290. 2024-11-20T09:02:03,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290. as already flushing 2024-11-20T09:02:03,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290. 2024-11-20T09:02:03,132 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742423_1599 (size=12151) 2024-11-20T09:02:03,132 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 2efe576d1bb7feeeb5d5727034c1a290, NAME => 'TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:02:03,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 2efe576d1bb7feeeb5d5727034c1a290, NAME => 'TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:02:03,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2efe576d1bb7feeeb5d5727034c1a290, NAME => 'TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2efe576d1bb7feeeb5d5727034c1a290, NAME => 'TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:02:03,140 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=159 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/.tmp/C/054ab2d467da47b48610e6b11130a4d4 2024-11-20T09:02:03,143 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/.tmp/A/39631abd030a4b0389e9a5a04a1234e6 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/A/39631abd030a4b0389e9a5a04a1234e6 2024-11-20T09:02:03,147 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/A/39631abd030a4b0389e9a5a04a1234e6, entries=200, sequenceid=159, filesize=38.8 K 2024-11-20T09:02:03,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-20T09:02:03,148 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/.tmp/B/ff0108518d3f4698a41614145284afb9 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/B/ff0108518d3f4698a41614145284afb9 2024-11-20T09:02:03,151 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/B/ff0108518d3f4698a41614145284afb9, entries=150, sequenceid=159, filesize=11.9 K 2024-11-20T09:02:03,152 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/.tmp/C/054ab2d467da47b48610e6b11130a4d4 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/C/054ab2d467da47b48610e6b11130a4d4 2024-11-20T09:02:03,155 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/C/054ab2d467da47b48610e6b11130a4d4, entries=150, sequenceid=159, filesize=11.9 K 2024-11-20T09:02:03,156 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=40.25 KB/41220 for 2efe576d1bb7feeeb5d5727034c1a290 in 874ms, sequenceid=159, compaction requested=true 2024-11-20T09:02:03,156 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2efe576d1bb7feeeb5d5727034c1a290: 2024-11-20T09:02:03,156 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2efe576d1bb7feeeb5d5727034c1a290:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T09:02:03,156 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T09:02:03,156 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T09:02:03,156 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T09:02:03,156 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2efe576d1bb7feeeb5d5727034c1a290:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T09:02:03,156 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T09:02:03,156 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2efe576d1bb7feeeb5d5727034c1a290:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T09:02:03,156 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T09:02:03,158 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101899 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T09:02:03,158 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HStore(1541): 2efe576d1bb7feeeb5d5727034c1a290/A is initiating minor compaction (all files) 2024-11-20T09:02:03,158 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2efe576d1bb7feeeb5d5727034c1a290/A in TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290. 2024-11-20T09:02:03,158 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/A/f288fe87ca7141bf99732dfa7941b872, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/A/056d297ddd93429fbc52fefff1e2fee0, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/A/39631abd030a4b0389e9a5a04a1234e6] into tmpdir=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/.tmp, totalSize=99.5 K 2024-11-20T09:02:03,158 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290. 2024-11-20T09:02:03,158 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290. files: [hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/A/f288fe87ca7141bf99732dfa7941b872, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/A/056d297ddd93429fbc52fefff1e2fee0, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/A/39631abd030a4b0389e9a5a04a1234e6] 2024-11-20T09:02:03,158 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36393 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T09:02:03,158 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1541): 2efe576d1bb7feeeb5d5727034c1a290/B is initiating minor compaction (all files) 2024-11-20T09:02:03,158 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2efe576d1bb7feeeb5d5727034c1a290/B in TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290. 2024-11-20T09:02:03,158 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/B/9ca1284dbf6842f1a7001bfe7ba44eeb, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/B/544630a02cdf4b11a238078ccee09bf1, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/B/ff0108518d3f4698a41614145284afb9] into tmpdir=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/.tmp, totalSize=35.5 K 2024-11-20T09:02:03,159 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.Compactor(225): Compacting f288fe87ca7141bf99732dfa7941b872, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1732093318510 2024-11-20T09:02:03,159 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting 9ca1284dbf6842f1a7001bfe7ba44eeb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1732093318510 2024-11-20T09:02:03,159 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting 544630a02cdf4b11a238078ccee09bf1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1732093319689 2024-11-20T09:02:03,160 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.Compactor(225): Compacting 056d297ddd93429fbc52fefff1e2fee0, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1732093319689 2024-11-20T09:02:03,160 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.Compactor(225): Compacting 39631abd030a4b0389e9a5a04a1234e6, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1732093321946 2024-11-20T09:02:03,160 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting ff0108518d3f4698a41614145284afb9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1732093321946 2024-11-20T09:02:03,168 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=2efe576d1bb7feeeb5d5727034c1a290] 2024-11-20T09:02:03,169 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2efe576d1bb7feeeb5d5727034c1a290#B#compaction#509 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-20T09:02:03,169 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/.tmp/B/258b6dbd94b94f218dca88b140d529f8 is 50, key is test_row_0/B:col10/1732093322280/Put/seqid=0 2024-11-20T09:02:03,171 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411209840c85ca7af419fadf322b8d0ec4a31_2efe576d1bb7feeeb5d5727034c1a290 store=[table=TestAcidGuarantees family=A region=2efe576d1bb7feeeb5d5727034c1a290] 2024-11-20T09:02:03,173 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411209840c85ca7af419fadf322b8d0ec4a31_2efe576d1bb7feeeb5d5727034c1a290, store=[table=TestAcidGuarantees family=A region=2efe576d1bb7feeeb5d5727034c1a290] 2024-11-20T09:02:03,173 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411209840c85ca7af419fadf322b8d0ec4a31_2efe576d1bb7feeeb5d5727034c1a290 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=2efe576d1bb7feeeb5d5727034c1a290] 2024-11-20T09:02:03,175 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742424_1600 (size=12493) 2024-11-20T09:02:03,180 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/.tmp/B/258b6dbd94b94f218dca88b140d529f8 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/B/258b6dbd94b94f218dca88b140d529f8 2024-11-20T09:02:03,184 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2efe576d1bb7feeeb5d5727034c1a290/B of 2efe576d1bb7feeeb5d5727034c1a290 into 258b6dbd94b94f218dca88b140d529f8(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T09:02:03,184 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2efe576d1bb7feeeb5d5727034c1a290: 2024-11-20T09:02:03,184 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., storeName=2efe576d1bb7feeeb5d5727034c1a290/B, priority=13, startTime=1732093323156; duration=0sec 2024-11-20T09:02:03,184 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T09:02:03,184 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2efe576d1bb7feeeb5d5727034c1a290:B 2024-11-20T09:02:03,184 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T09:02:03,186 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36393 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T09:02:03,186 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1541): 2efe576d1bb7feeeb5d5727034c1a290/C is initiating minor compaction (all files) 2024-11-20T09:02:03,186 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2efe576d1bb7feeeb5d5727034c1a290/C in TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290. 2024-11-20T09:02:03,186 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/C/f118f2f791014323ba76947729c2bf22, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/C/4df61a541594426a8a3d5dbe15cf7647, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/C/054ab2d467da47b48610e6b11130a4d4] into tmpdir=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/.tmp, totalSize=35.5 K 2024-11-20T09:02:03,186 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting f118f2f791014323ba76947729c2bf22, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1732093318510 2024-11-20T09:02:03,187 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting 4df61a541594426a8a3d5dbe15cf7647, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1732093319689 2024-11-20T09:02:03,187 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting 054ab2d467da47b48610e6b11130a4d4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1732093321946 2024-11-20T09:02:03,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742425_1601 (size=4469) 2024-11-20T09:02:03,206 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2efe576d1bb7feeeb5d5727034c1a290#C#compaction#511 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-20T09:02:03,206 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/.tmp/C/bc89334ca1e245e0a7d498e1ba63e8b9 is 50, key is test_row_0/C:col10/1732093322280/Put/seqid=0 2024-11-20T09:02:03,214 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742426_1602 (size=12493) 2024-11-20T09:02:03,222 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/.tmp/C/bc89334ca1e245e0a7d498e1ba63e8b9 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/C/bc89334ca1e245e0a7d498e1ba63e8b9 2024-11-20T09:02:03,229 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2efe576d1bb7feeeb5d5727034c1a290/C of 2efe576d1bb7feeeb5d5727034c1a290 into bc89334ca1e245e0a7d498e1ba63e8b9(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T09:02:03,229 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2efe576d1bb7feeeb5d5727034c1a290: 2024-11-20T09:02:03,229 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., storeName=2efe576d1bb7feeeb5d5727034c1a290/C, priority=13, startTime=1732093323156; duration=0sec 2024-11-20T09:02:03,229 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T09:02:03,229 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2efe576d1bb7feeeb5d5727034c1a290:C 2024-11-20T09:02:03,285 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-20T09:02:03,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290. 2024-11-20T09:02:03,286 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2902): Flushing 2efe576d1bb7feeeb5d5727034c1a290 3/3 column families, dataSize=40.25 KB heapSize=106.22 KB 2024-11-20T09:02:03,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411204fdc260e158c4312a01c7eed60f59740_2efe576d1bb7feeeb5d5727034c1a290 is 50, key is test_row_0/A:col10/1732093322291/Put/seqid=0 2024-11-20T09:02:03,298 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742427_1603 (size=12304) 2024-11-20T09:02:03,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(8855): Flush requested on 2efe576d1bb7feeeb5d5727034c1a290 2024-11-20T09:02:03,450 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290. as already flushing 2024-11-20T09:02:03,552 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:02:03,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53418 deadline: 1732093383548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:02:03,552 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:02:03,552 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:02:03,552 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:02:03,596 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2efe576d1bb7feeeb5d5727034c1a290#A#compaction#510 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T09:02:03,596 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/.tmp/A/097bb2dc8c6943b2bce068d498b839bd is 175, key is test_row_0/A:col10/1732093322280/Put/seqid=0 2024-11-20T09:02:03,604 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742428_1604 (size=31447) 2024-11-20T09:02:03,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-20T09:02:03,660 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:02:03,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53418 deadline: 1732093383657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:02:03,661 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:02:03,661 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:02:03,661 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:02:03,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:03,703 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411204fdc260e158c4312a01c7eed60f59740_2efe576d1bb7feeeb5d5727034c1a290 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411204fdc260e158c4312a01c7eed60f59740_2efe576d1bb7feeeb5d5727034c1a290 2024-11-20T09:02:03,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/.tmp/A/331981096486447898ee96d3c3d23a2a, store: [table=TestAcidGuarantees family=A region=2efe576d1bb7feeeb5d5727034c1a290] 2024-11-20T09:02:03,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/.tmp/A/331981096486447898ee96d3c3d23a2a is 175, key is test_row_0/A:col10/1732093322291/Put/seqid=0 2024-11-20T09:02:03,708 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742429_1605 (size=31105) 2024-11-20T09:02:03,870 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:02:03,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53418 deadline: 1732093383867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:02:03,870 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:02:03,870 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:02:03,870 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:02:04,008 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/.tmp/A/097bb2dc8c6943b2bce068d498b839bd as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/A/097bb2dc8c6943b2bce068d498b839bd 2024-11-20T09:02:04,012 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2efe576d1bb7feeeb5d5727034c1a290/A of 2efe576d1bb7feeeb5d5727034c1a290 into 097bb2dc8c6943b2bce068d498b839bd(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T09:02:04,012 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2efe576d1bb7feeeb5d5727034c1a290: 2024-11-20T09:02:04,012 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., storeName=2efe576d1bb7feeeb5d5727034c1a290/A, priority=13, startTime=1732093323156; duration=0sec 2024-11-20T09:02:04,012 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T09:02:04,012 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2efe576d1bb7feeeb5d5727034c1a290:A 2024-11-20T09:02:04,115 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=170, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/.tmp/A/331981096486447898ee96d3c3d23a2a 2024-11-20T09:02:04,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/.tmp/B/0e93702288784452abdf98d200cdb5c1 is 50, key is test_row_0/B:col10/1732093322291/Put/seqid=0 2024-11-20T09:02:04,124 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742430_1606 (size=12151) 2024-11-20T09:02:04,178 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:02:04,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53418 deadline: 1732093384177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:02:04,178 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:02:04,178 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:02:04,179 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:02:04,525 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/.tmp/B/0e93702288784452abdf98d200cdb5c1 2024-11-20T09:02:04,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/.tmp/C/f2d0e864812f432f855b5de552fc6cc4 is 50, key is test_row_0/C:col10/1732093322291/Put/seqid=0 2024-11-20T09:02:04,539 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742431_1607 (size=12151) 2024-11-20T09:02:04,601 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:02:04,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53442 deadline: 1732093384597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:02:04,601 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:02:04,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53372 deadline: 1732093384597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:02:04,601 WARN [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRpcRetryingCaller(177): Call to be1eb2620e0e,34223,1732093159960 for 'test_row_1' in 2efe576d1bb7feeeb5d5727034c1a290 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8196 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-20T09:02:04,602 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:02:04,602 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:02:04,602 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to be1eb2620e0e,34223,1732093159960 for 'test_row_0' in 2efe576d1bb7feeeb5d5727034c1a290 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8196 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-20T09:02:04,602 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:02:04,602 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:02:04,602 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:02:04,602 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:02:04,610 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:02:04,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53462 deadline: 1732093384607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:02:04,610 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to be1eb2620e0e,34223,1732093159960 for 'test_row_1' in 2efe576d1bb7feeeb5d5727034c1a290 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8203 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-20T09:02:04,611 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:02:04,611 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:02:04,611 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:02:04,619 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:02:04,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53446 deadline: 1732093384617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:02:04,619 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to be1eb2620e0e,34223,1732093159960 for 'test_row_1' in 2efe576d1bb7feeeb5d5727034c1a290 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8212 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-20T09:02:04,619 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:02:04,619 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:02:04,620 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:02:04,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-20T09:02:04,689 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:02:04,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53418 deadline: 1732093384687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:02:04,690 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:02:04,690 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:02:04,690 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:02:04,940 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/.tmp/C/f2d0e864812f432f855b5de552fc6cc4 2024-11-20T09:02:04,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/.tmp/A/331981096486447898ee96d3c3d23a2a as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/A/331981096486447898ee96d3c3d23a2a 2024-11-20T09:02:04,947 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/A/331981096486447898ee96d3c3d23a2a, entries=150, sequenceid=170, filesize=30.4 K 2024-11-20T09:02:04,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/.tmp/B/0e93702288784452abdf98d200cdb5c1 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/B/0e93702288784452abdf98d200cdb5c1 2024-11-20T09:02:04,951 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/B/0e93702288784452abdf98d200cdb5c1, entries=150, sequenceid=170, filesize=11.9 K 2024-11-20T09:02:04,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/.tmp/C/f2d0e864812f432f855b5de552fc6cc4 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/C/f2d0e864812f432f855b5de552fc6cc4 2024-11-20T09:02:04,954 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/C/f2d0e864812f432f855b5de552fc6cc4, entries=150, sequenceid=170, filesize=11.9 K 2024-11-20T09:02:04,955 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~106.17 KB/108720, currentSize=161.02 KB/164880 for 2efe576d1bb7feeeb5d5727034c1a290 in 1670ms, sequenceid=170, compaction requested=false 2024-11-20T09:02:04,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2603): Flush status journal for 2efe576d1bb7feeeb5d5727034c1a290: 2024-11-20T09:02:04,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290. 2024-11-20T09:02:04,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=153 2024-11-20T09:02:04,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.HMaster(4169): Remote procedure done, pid=153 2024-11-20T09:02:04,958 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=153, resume processing ppid=152 2024-11-20T09:02:04,958 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=153, ppid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4360 sec 2024-11-20T09:02:04,960 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees in 2.4410 sec 2024-11-20T09:02:05,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(8855): Flush requested on 2efe576d1bb7feeeb5d5727034c1a290 2024-11-20T09:02:05,709 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T09:02:05,710 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2efe576d1bb7feeeb5d5727034c1a290 3/3 column families, dataSize=167.72 KB heapSize=440.20 KB 2024-11-20T09:02:05,716 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112067c9ca74eebd42beb616958879ccb8e6_2efe576d1bb7feeeb5d5727034c1a290 is 50, key is test_row_0/A:col10/1732093325708/Put/seqid=0 2024-11-20T09:02:05,719 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742432_1608 (size=12304) 2024-11-20T09:02:05,720 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:05,732 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112067c9ca74eebd42beb616958879ccb8e6_2efe576d1bb7feeeb5d5727034c1a290 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112067c9ca74eebd42beb616958879ccb8e6_2efe576d1bb7feeeb5d5727034c1a290 2024-11-20T09:02:05,733 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/.tmp/A/8bcffbd0558c4204b995c949a27bed12, store: [table=TestAcidGuarantees family=A region=2efe576d1bb7feeeb5d5727034c1a290] 2024-11-20T09:02:05,733 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/.tmp/A/8bcffbd0558c4204b995c949a27bed12 is 175, key is test_row_0/A:col10/1732093325708/Put/seqid=0 2024-11-20T09:02:05,733 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:02:05,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53418 deadline: 1732093385730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:02:05,734 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:02:05,734 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:02:05,734 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:02:05,736 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742433_1609 (size=31105) 2024-11-20T09:02:05,839 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:02:05,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53418 deadline: 1732093385837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:02:05,840 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:02:05,840 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:02:05,840 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:02:06,048 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:02:06,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53418 deadline: 1732093386046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:02:06,049 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:02:06,049 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:02:06,049 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:02:06,137 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=199, memsize=55.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/.tmp/A/8bcffbd0558c4204b995c949a27bed12 2024-11-20T09:02:06,144 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/.tmp/B/e13fd36033ce44ff8a864dbbf0ed2b5b is 50, key is test_row_0/B:col10/1732093325708/Put/seqid=0 2024-11-20T09:02:06,147 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742434_1610 (size=12151) 2024-11-20T09:02:06,148 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=199 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/.tmp/B/e13fd36033ce44ff8a864dbbf0ed2b5b 2024-11-20T09:02:06,155 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/.tmp/C/f1cd2c657db84a638ce6f74ac6471683 is 50, key is test_row_0/C:col10/1732093325708/Put/seqid=0 2024-11-20T09:02:06,160 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742435_1611 (size=12151) 2024-11-20T09:02:06,161 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=199 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/.tmp/C/f1cd2c657db84a638ce6f74ac6471683 2024-11-20T09:02:06,165 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/.tmp/A/8bcffbd0558c4204b995c949a27bed12 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/A/8bcffbd0558c4204b995c949a27bed12 2024-11-20T09:02:06,169 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/A/8bcffbd0558c4204b995c949a27bed12, entries=150, sequenceid=199, filesize=30.4 K 2024-11-20T09:02:06,170 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/.tmp/B/e13fd36033ce44ff8a864dbbf0ed2b5b as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/B/e13fd36033ce44ff8a864dbbf0ed2b5b 2024-11-20T09:02:06,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,173 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/B/e13fd36033ce44ff8a864dbbf0ed2b5b, entries=150, sequenceid=199, filesize=11.9 K 2024-11-20T09:02:06,174 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/.tmp/C/f1cd2c657db84a638ce6f74ac6471683 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/C/f1cd2c657db84a638ce6f74ac6471683 2024-11-20T09:02:06,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,178 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/C/f1cd2c657db84a638ce6f74ac6471683, entries=150, sequenceid=199, filesize=11.9 K 2024-11-20T09:02:06,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,179 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~167.72 KB/171750, heapSize ~440.16 KB/450720, currentSize=33.54 KB/34350 for 2efe576d1bb7feeeb5d5727034c1a290 in 469ms, sequenceid=199, compaction requested=true 2024-11-20T09:02:06,179 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2efe576d1bb7feeeb5d5727034c1a290: 2024-11-20T09:02:06,179 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2efe576d1bb7feeeb5d5727034c1a290:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T09:02:06,179 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T09:02:06,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,179 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2efe576d1bb7feeeb5d5727034c1a290:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T09:02:06,179 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T09:02:06,179 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T09:02:06,179 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2efe576d1bb7feeeb5d5727034c1a290:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T09:02:06,179 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T09:02:06,179 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T09:02:06,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,181 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T09:02:06,181 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93657 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T09:02:06,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,181 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1541): 2efe576d1bb7feeeb5d5727034c1a290/B is initiating minor compaction (all files) 2024-11-20T09:02:06,181 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HStore(1541): 2efe576d1bb7feeeb5d5727034c1a290/A is initiating minor compaction (all files) 2024-11-20T09:02:06,181 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2efe576d1bb7feeeb5d5727034c1a290/A in TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290. 2024-11-20T09:02:06,181 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2efe576d1bb7feeeb5d5727034c1a290/B in TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290. 2024-11-20T09:02:06,181 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/A/097bb2dc8c6943b2bce068d498b839bd, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/A/331981096486447898ee96d3c3d23a2a, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/A/8bcffbd0558c4204b995c949a27bed12] into tmpdir=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/.tmp, totalSize=91.5 K 2024-11-20T09:02:06,181 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/B/258b6dbd94b94f218dca88b140d529f8, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/B/0e93702288784452abdf98d200cdb5c1, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/B/e13fd36033ce44ff8a864dbbf0ed2b5b] into tmpdir=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/.tmp, totalSize=35.9 K 2024-11-20T09:02:06,181 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290. 2024-11-20T09:02:06,181 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290. files: [hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/A/097bb2dc8c6943b2bce068d498b839bd, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/A/331981096486447898ee96d3c3d23a2a, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/A/8bcffbd0558c4204b995c949a27bed12] 2024-11-20T09:02:06,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,181 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.Compactor(225): Compacting 097bb2dc8c6943b2bce068d498b839bd, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1732093321946 2024-11-20T09:02:06,181 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting 258b6dbd94b94f218dca88b140d529f8, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1732093321946 2024-11-20T09:02:06,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,182 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting 0e93702288784452abdf98d200cdb5c1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1732093322291 2024-11-20T09:02:06,182 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.Compactor(225): Compacting 331981096486447898ee96d3c3d23a2a, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1732093322291 2024-11-20T09:02:06,182 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting e13fd36033ce44ff8a864dbbf0ed2b5b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1732093323541 2024-11-20T09:02:06,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,182 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8bcffbd0558c4204b995c949a27bed12, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1732093323541 2024-11-20T09:02:06,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,190 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=2efe576d1bb7feeeb5d5727034c1a290] 2024-11-20T09:02:06,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,190 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2efe576d1bb7feeeb5d5727034c1a290#B#compaction#518 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-20T09:02:06,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,191 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/.tmp/B/082f3d32b8fa48ba90c235e5e2034313 is 50, key is test_row_0/B:col10/1732093325708/Put/seqid=0 2024-11-20T09:02:06,191 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241120a7ff1f9426c746b185dec1c614164b36_2efe576d1bb7feeeb5d5727034c1a290 store=[table=TestAcidGuarantees family=A region=2efe576d1bb7feeeb5d5727034c1a290] 2024-11-20T09:02:06,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,193 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241120a7ff1f9426c746b185dec1c614164b36_2efe576d1bb7feeeb5d5727034c1a290, store=[table=TestAcidGuarantees family=A region=2efe576d1bb7feeeb5d5727034c1a290] 2024-11-20T09:02:06,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,193 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120a7ff1f9426c746b185dec1c614164b36_2efe576d1bb7feeeb5d5727034c1a290 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=2efe576d1bb7feeeb5d5727034c1a290] 2024-11-20T09:02:06,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,195 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742436_1612 (size=12595) 2024-11-20T09:02:06,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,197 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742437_1613 (size=4469) 2024-11-20T09:02:06,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,198 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2efe576d1bb7feeeb5d5727034c1a290#A#compaction#519 average throughput is 3.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T09:02:06,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,199 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/.tmp/A/323f78dd4aa2450b8f92bb0d208bc874 is 175, key is test_row_0/A:col10/1732093325708/Put/seqid=0 2024-11-20T09:02:06,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,200 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/.tmp/B/082f3d32b8fa48ba90c235e5e2034313 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/B/082f3d32b8fa48ba90c235e5e2034313 2024-11-20T09:02:06,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,202 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742438_1614 (size=31549) 2024-11-20T09:02:06,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,205 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2efe576d1bb7feeeb5d5727034c1a290/B of 2efe576d1bb7feeeb5d5727034c1a290 into 082f3d32b8fa48ba90c235e5e2034313(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T09:02:06,205 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2efe576d1bb7feeeb5d5727034c1a290: 2024-11-20T09:02:06,205 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., storeName=2efe576d1bb7feeeb5d5727034c1a290/B, priority=13, startTime=1732093326179; duration=0sec 2024-11-20T09:02:06,205 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T09:02:06,205 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2efe576d1bb7feeeb5d5727034c1a290:B 2024-11-20T09:02:06,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,205 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T09:02:06,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,207 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T09:02:06,207 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1541): 2efe576d1bb7feeeb5d5727034c1a290/C is initiating minor compaction (all files) 2024-11-20T09:02:06,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,207 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2efe576d1bb7feeeb5d5727034c1a290/C in TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290. 2024-11-20T09:02:06,207 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/C/bc89334ca1e245e0a7d498e1ba63e8b9, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/C/f2d0e864812f432f855b5de552fc6cc4, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/C/f1cd2c657db84a638ce6f74ac6471683] into tmpdir=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/.tmp, totalSize=35.9 K 2024-11-20T09:02:06,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,208 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting bc89334ca1e245e0a7d498e1ba63e8b9, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1732093321946 2024-11-20T09:02:06,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,208 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/.tmp/A/323f78dd4aa2450b8f92bb0d208bc874 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/A/323f78dd4aa2450b8f92bb0d208bc874 2024-11-20T09:02:06,208 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting f2d0e864812f432f855b5de552fc6cc4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1732093322291 2024-11-20T09:02:06,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,209 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting f1cd2c657db84a638ce6f74ac6471683, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1732093323541 2024-11-20T09:02:06,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,214 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2efe576d1bb7feeeb5d5727034c1a290/A of 2efe576d1bb7feeeb5d5727034c1a290 into 323f78dd4aa2450b8f92bb0d208bc874(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T09:02:06,214 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2efe576d1bb7feeeb5d5727034c1a290: 2024-11-20T09:02:06,214 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., storeName=2efe576d1bb7feeeb5d5727034c1a290/A, priority=13, startTime=1732093326179; duration=0sec 2024-11-20T09:02:06,214 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T09:02:06,214 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2efe576d1bb7feeeb5d5727034c1a290:A 2024-11-20T09:02:06,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,218 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2efe576d1bb7feeeb5d5727034c1a290#C#compaction#520 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T09:02:06,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,218 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/.tmp/C/11b8f70b90d745fba44537b31120d560 is 50, key is test_row_0/C:col10/1732093325708/Put/seqid=0 2024-11-20T09:02:06,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742439_1615 (size=12595) 2024-11-20T09:02:06,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,239 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/.tmp/C/11b8f70b90d745fba44537b31120d560 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/C/11b8f70b90d745fba44537b31120d560 2024-11-20T09:02:06,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,243 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2efe576d1bb7feeeb5d5727034c1a290/C of 2efe576d1bb7feeeb5d5727034c1a290 into 11b8f70b90d745fba44537b31120d560(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T09:02:06,243 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2efe576d1bb7feeeb5d5727034c1a290: 2024-11-20T09:02:06,243 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., storeName=2efe576d1bb7feeeb5d5727034c1a290/C, priority=13, startTime=1732093326179; duration=0sec 2024-11-20T09:02:06,243 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T09:02:06,243 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2efe576d1bb7feeeb5d5727034c1a290:C 2024-11-20T09:02:06,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(8855): Flush requested on 2efe576d1bb7feeeb5d5727034c1a290 2024-11-20T09:02:06,366 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T09:02:06,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,366 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2efe576d1bb7feeeb5d5727034c1a290 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-20T09:02:06,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,373 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120efbe038c59774ee4b0f1b4620ddcc8fe_2efe576d1bb7feeeb5d5727034c1a290 is 50, key is test_row_0/A:col10/1732093326365/Put/seqid=0 2024-11-20T09:02:06,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,377 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742440_1616 (size=14794) 2024-11-20T09:02:06,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,460 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:02:06,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53418 deadline: 1732093386456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:02:06,460 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:02:06,460 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:02:06,460 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:02:06,567 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:02:06,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53418 deadline: 1732093386567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:02:06,568 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:02:06,568 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:02:06,568 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:02:06,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-20T09:02:06,677 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T09:02:06,677 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T09:02:06,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] procedure2.ProcedureExecutor(1139): Stored pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees 2024-11-20T09:02:06,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-20T09:02:06,679 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T09:02:06,680 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T09:02:06,680 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=155, ppid=154, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T09:02:06,777 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:06,779 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:02:06,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53418 deadline: 1732093386777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:02:06,779 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:02:06,779 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:02:06,779 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:02:06,781 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120efbe038c59774ee4b0f1b4620ddcc8fe_2efe576d1bb7feeeb5d5727034c1a290 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120efbe038c59774ee4b0f1b4620ddcc8fe_2efe576d1bb7feeeb5d5727034c1a290 2024-11-20T09:02:06,781 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/.tmp/A/70a78ff38a8e4f509ca1ca1468cd5980, store: [table=TestAcidGuarantees family=A region=2efe576d1bb7feeeb5d5727034c1a290] 2024-11-20T09:02:06,782 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/.tmp/A/70a78ff38a8e4f509ca1ca1468cd5980 is 175, key is test_row_0/A:col10/1732093326365/Put/seqid=0 2024-11-20T09:02:06,785 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742441_1617 (size=39749) 2024-11-20T09:02:06,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-20T09:02:06,831 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-20T09:02:06,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290. 2024-11-20T09:02:06,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290. as already flushing 2024-11-20T09:02:06,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290. 2024-11-20T09:02:06,832 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 2efe576d1bb7feeeb5d5727034c1a290, NAME => 'TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:02:06,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 2efe576d1bb7feeeb5d5727034c1a290, NAME => 'TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:02:06,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2efe576d1bb7feeeb5d5727034c1a290, NAME => 'TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2efe576d1bb7feeeb5d5727034c1a290, NAME => 'TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:02:06,984 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-20T09:02:06,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290. 2024-11-20T09:02:06,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290. as already flushing 2024-11-20T09:02:06,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290. 2024-11-20T09:02:06,985 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 2efe576d1bb7feeeb5d5727034c1a290, NAME => 'TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:02:06,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 2efe576d1bb7feeeb5d5727034c1a290, NAME => 'TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:02:06,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2efe576d1bb7feeeb5d5727034c1a290, NAME => 'TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2efe576d1bb7feeeb5d5727034c1a290, NAME => 'TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:02:06,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-20T09:02:07,090 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:02:07,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53418 deadline: 1732093387087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:02:07,090 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:02:07,090 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:02:07,090 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:02:07,137 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-20T09:02:07,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290. 2024-11-20T09:02:07,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290. as already flushing 2024-11-20T09:02:07,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290. 2024-11-20T09:02:07,138 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 2efe576d1bb7feeeb5d5727034c1a290, NAME => 'TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:02:07,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 2efe576d1bb7feeeb5d5727034c1a290, NAME => 'TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:02:07,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2efe576d1bb7feeeb5d5727034c1a290, NAME => 'TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2efe576d1bb7feeeb5d5727034c1a290, NAME => 'TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:02:07,186 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=213, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/.tmp/A/70a78ff38a8e4f509ca1ca1468cd5980 2024-11-20T09:02:07,191 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/.tmp/B/0f0f6b0213584768b430eb41475e8182 is 50, key is test_row_0/B:col10/1732093326365/Put/seqid=0 2024-11-20T09:02:07,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742442_1618 (size=12151) 2024-11-20T09:02:07,290 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-20T09:02:07,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290. 2024-11-20T09:02:07,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290. as already flushing 2024-11-20T09:02:07,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290. 2024-11-20T09:02:07,291 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 2efe576d1bb7feeeb5d5727034c1a290, NAME => 'TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:02:07,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 2efe576d1bb7feeeb5d5727034c1a290, NAME => 'TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:02:07,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2efe576d1bb7feeeb5d5727034c1a290, NAME => 'TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2efe576d1bb7feeeb5d5727034c1a290, NAME => 'TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:02:07,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-20T09:02:07,443 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-20T09:02:07,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290. 2024-11-20T09:02:07,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290. as already flushing 2024-11-20T09:02:07,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290. 2024-11-20T09:02:07,444 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 2efe576d1bb7feeeb5d5727034c1a290, NAME => 'TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:02:07,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 2efe576d1bb7feeeb5d5727034c1a290, NAME => 'TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:02:07,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2efe576d1bb7feeeb5d5727034c1a290, NAME => 'TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2efe576d1bb7feeeb5d5727034c1a290, NAME => 'TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:02:07,595 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=213 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/.tmp/B/0f0f6b0213584768b430eb41475e8182 2024-11-20T09:02:07,596 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-20T09:02:07,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290. 2024-11-20T09:02:07,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290. as already flushing 2024-11-20T09:02:07,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290. 2024-11-20T09:02:07,597 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 2efe576d1bb7feeeb5d5727034c1a290, NAME => 'TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:02:07,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 2efe576d1bb7feeeb5d5727034c1a290, NAME => 'TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:02:07,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2efe576d1bb7feeeb5d5727034c1a290, NAME => 'TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2efe576d1bb7feeeb5d5727034c1a290, NAME => 'TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:02:07,600 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:02:07,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53418 deadline: 1732093387597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:02:07,600 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:02:07,600 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:02:07,600 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:02:07,602 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/.tmp/C/e521a000085e40e9a7574910a2b42c19 is 50, key is test_row_0/C:col10/1732093326365/Put/seqid=0 2024-11-20T09:02:07,606 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742443_1619 (size=12151) 2024-11-20T09:02:07,749 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-20T09:02:07,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290. 2024-11-20T09:02:07,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290. as already flushing 2024-11-20T09:02:07,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290. 2024-11-20T09:02:07,750 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 2efe576d1bb7feeeb5d5727034c1a290, NAME => 'TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:02:07,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 2efe576d1bb7feeeb5d5727034c1a290, NAME => 'TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:02:07,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2efe576d1bb7feeeb5d5727034c1a290, NAME => 'TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2efe576d1bb7feeeb5d5727034c1a290, NAME => 'TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:02:07,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-20T09:02:07,902 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-20T09:02:07,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290. 2024-11-20T09:02:07,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290. as already flushing 2024-11-20T09:02:07,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290. 2024-11-20T09:02:07,903 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 2efe576d1bb7feeeb5d5727034c1a290, NAME => 'TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:02:07,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 2efe576d1bb7feeeb5d5727034c1a290, NAME => 'TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:02:07,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2efe576d1bb7feeeb5d5727034c1a290, NAME => 'TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2efe576d1bb7feeeb5d5727034c1a290, NAME => 'TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:02:08,006 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=213 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/.tmp/C/e521a000085e40e9a7574910a2b42c19 2024-11-20T09:02:08,010 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/.tmp/A/70a78ff38a8e4f509ca1ca1468cd5980 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/A/70a78ff38a8e4f509ca1ca1468cd5980 2024-11-20T09:02:08,013 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/A/70a78ff38a8e4f509ca1ca1468cd5980, entries=200, sequenceid=213, filesize=38.8 K 2024-11-20T09:02:08,014 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/.tmp/B/0f0f6b0213584768b430eb41475e8182 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/B/0f0f6b0213584768b430eb41475e8182 2024-11-20T09:02:08,017 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/B/0f0f6b0213584768b430eb41475e8182, entries=150, sequenceid=213, filesize=11.9 K 2024-11-20T09:02:08,017 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/.tmp/C/e521a000085e40e9a7574910a2b42c19 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/C/e521a000085e40e9a7574910a2b42c19 2024-11-20T09:02:08,020 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/C/e521a000085e40e9a7574910a2b42c19, entries=150, sequenceid=213, filesize=11.9 K 2024-11-20T09:02:08,021 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 2efe576d1bb7feeeb5d5727034c1a290 in 1655ms, sequenceid=213, compaction requested=false 2024-11-20T09:02:08,021 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2efe576d1bb7feeeb5d5727034c1a290: 2024-11-20T09:02:08,055 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-20T09:02:08,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290. 2024-11-20T09:02:08,056 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2902): Flushing 2efe576d1bb7feeeb5d5727034c1a290 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-20T09:02:08,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112060b1a264ae9140d9bc4e841e5f0c6630_2efe576d1bb7feeeb5d5727034c1a290 is 50, key is test_row_0/A:col10/1732093326455/Put/seqid=0 2024-11-20T09:02:08,072 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742444_1620 (size=12304) 2024-11-20T09:02:08,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:08,476 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112060b1a264ae9140d9bc4e841e5f0c6630_2efe576d1bb7feeeb5d5727034c1a290 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112060b1a264ae9140d9bc4e841e5f0c6630_2efe576d1bb7feeeb5d5727034c1a290 2024-11-20T09:02:08,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/.tmp/A/6850b05bd9bd49bca3bdcc0de66fcd9a, store: [table=TestAcidGuarantees family=A region=2efe576d1bb7feeeb5d5727034c1a290] 2024-11-20T09:02:08,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/.tmp/A/6850b05bd9bd49bca3bdcc0de66fcd9a is 175, key is test_row_0/A:col10/1732093326455/Put/seqid=0 2024-11-20T09:02:08,481 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742445_1621 (size=31105) 2024-11-20T09:02:08,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(8855): Flush requested on 2efe576d1bb7feeeb5d5727034c1a290 2024-11-20T09:02:08,620 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290. as already flushing 2024-11-20T09:02:08,650 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:02:08,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53418 deadline: 1732093388648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:02:08,651 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:02:08,651 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:02:08,651 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:02:08,759 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:02:08,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53418 deadline: 1732093388757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:02:08,760 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:02:08,760 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:02:08,760 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:02:08,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-20T09:02:08,881 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=238, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/.tmp/A/6850b05bd9bd49bca3bdcc0de66fcd9a 2024-11-20T09:02:08,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/.tmp/B/a58f882ce9c544a3afb1e3f43869c330 is 50, key is test_row_0/B:col10/1732093326455/Put/seqid=0 2024-11-20T09:02:08,892 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742446_1622 (size=12151) 2024-11-20T09:02:08,969 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:02:08,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53418 deadline: 1732093388967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:02:08,969 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:02:08,970 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:02:08,970 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:02:09,278 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:02:09,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53418 deadline: 1732093389277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:02:09,279 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:02:09,279 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:02:09,279 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:02:09,293 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=238 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/.tmp/B/a58f882ce9c544a3afb1e3f43869c330 2024-11-20T09:02:09,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/.tmp/C/2a745a7663014d95a3952a396a494ba8 is 50, key is test_row_0/C:col10/1732093326455/Put/seqid=0 2024-11-20T09:02:09,320 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742447_1623 (size=12151) 2024-11-20T09:02:09,321 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=238 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/.tmp/C/2a745a7663014d95a3952a396a494ba8 2024-11-20T09:02:09,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/.tmp/A/6850b05bd9bd49bca3bdcc0de66fcd9a as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/A/6850b05bd9bd49bca3bdcc0de66fcd9a 2024-11-20T09:02:09,328 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/A/6850b05bd9bd49bca3bdcc0de66fcd9a, entries=150, sequenceid=238, filesize=30.4 K 2024-11-20T09:02:09,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/.tmp/B/a58f882ce9c544a3afb1e3f43869c330 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/B/a58f882ce9c544a3afb1e3f43869c330 2024-11-20T09:02:09,332 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/B/a58f882ce9c544a3afb1e3f43869c330, entries=150, sequenceid=238, filesize=11.9 K 2024-11-20T09:02:09,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/.tmp/C/2a745a7663014d95a3952a396a494ba8 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/C/2a745a7663014d95a3952a396a494ba8 2024-11-20T09:02:09,336 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/C/2a745a7663014d95a3952a396a494ba8, entries=150, sequenceid=238, filesize=11.9 K 2024-11-20T09:02:09,337 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 2efe576d1bb7feeeb5d5727034c1a290 in 1282ms, sequenceid=238, compaction requested=true 2024-11-20T09:02:09,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2603): Flush status journal for 2efe576d1bb7feeeb5d5727034c1a290: 2024-11-20T09:02:09,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290. 2024-11-20T09:02:09,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=155 2024-11-20T09:02:09,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.HMaster(4169): Remote procedure done, pid=155 2024-11-20T09:02:09,340 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=155, resume processing ppid=154 2024-11-20T09:02:09,340 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=155, ppid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6580 sec 2024-11-20T09:02:09,342 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees in 2.6640 sec 2024-11-20T09:02:09,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(8855): Flush requested on 2efe576d1bb7feeeb5d5727034c1a290 2024-11-20T09:02:09,790 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T09:02:09,790 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2efe576d1bb7feeeb5d5727034c1a290 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-20T09:02:09,797 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112026eb151bf6564975a36ee1a5429b4aef_2efe576d1bb7feeeb5d5727034c1a290 is 50, key is test_row_0/A:col10/1732093328646/Put/seqid=0 2024-11-20T09:02:09,806 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742448_1624 (size=12304) 2024-11-20T09:02:09,893 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:02:09,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 201 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53418 deadline: 1732093389889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:02:09,893 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:02:09,893 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:02:09,893 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:02:09,998 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:02:09,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 202 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53418 deadline: 1732093389996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:02:09,999 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:02:09,999 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:02:09,999 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:02:10,207 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:10,209 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:02:10,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 203 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53418 deadline: 1732093390207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:02:10,210 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:02:10,210 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:02:10,210 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:02:10,211 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112026eb151bf6564975a36ee1a5429b4aef_2efe576d1bb7feeeb5d5727034c1a290 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112026eb151bf6564975a36ee1a5429b4aef_2efe576d1bb7feeeb5d5727034c1a290 2024-11-20T09:02:10,211 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/.tmp/A/242ffce3ff2346b0af2cf7c7392478d0, store: [table=TestAcidGuarantees family=A region=2efe576d1bb7feeeb5d5727034c1a290] 2024-11-20T09:02:10,212 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/.tmp/A/242ffce3ff2346b0af2cf7c7392478d0 is 175, key is test_row_0/A:col10/1732093328646/Put/seqid=0 2024-11-20T09:02:10,215 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742449_1625 (size=31105) 2024-11-20T09:02:10,520 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:02:10,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 204 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53418 deadline: 1732093390517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:02:10,520 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:02:10,520 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:02:10,521 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:02:10,616 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=250, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/.tmp/A/242ffce3ff2346b0af2cf7c7392478d0 2024-11-20T09:02:10,623 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/.tmp/B/53b81d6b3e454d849ba4fe8c81a9021e is 50, key is test_row_0/B:col10/1732093328646/Put/seqid=0 2024-11-20T09:02:10,626 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742450_1626 (size=12151) 2024-11-20T09:02:10,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-20T09:02:10,847 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T09:02:10,848 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T09:02:10,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] procedure2.ProcedureExecutor(1139): Stored pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees 2024-11-20T09:02:10,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-20T09:02:10,850 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T09:02:10,850 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T09:02:10,850 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=157, ppid=156, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T09:02:10,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-20T09:02:11,002 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-20T09:02:11,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290. 2024-11-20T09:02:11,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290. as already flushing 2024-11-20T09:02:11,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290. 2024-11-20T09:02:11,003 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 2efe576d1bb7feeeb5d5727034c1a290, NAME => 'TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:02:11,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 2efe576d1bb7feeeb5d5727034c1a290, NAME => 'TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:02:11,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2efe576d1bb7feeeb5d5727034c1a290, NAME => 'TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2efe576d1bb7feeeb5d5727034c1a290, NAME => 'TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:02:11,027 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=250 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/.tmp/B/53b81d6b3e454d849ba4fe8c81a9021e 2024-11-20T09:02:11,029 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T09:02:11,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34223 {}] ipc.CallRunner(138): callId: 205 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53418 deadline: 1732093391027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 2024-11-20T09:02:11,029 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 , the old value is region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:02:11,029 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T09:02:11,029 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., hostname=be1eb2620e0e,34223,1732093159960, seqNum=5 because the exception is null or not the one we care about 2024-11-20T09:02:11,033 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/.tmp/C/148b1d5802434405a079fe65f4ef5cb4 is 50, key is test_row_0/C:col10/1732093328646/Put/seqid=0 2024-11-20T09:02:11,036 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742451_1627 (size=12151) 2024-11-20T09:02:11,155 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-20T09:02:11,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290. 2024-11-20T09:02:11,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290. as already flushing 2024-11-20T09:02:11,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290. 2024-11-20T09:02:11,156 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 2efe576d1bb7feeeb5d5727034c1a290, NAME => 'TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:02:11,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 2efe576d1bb7feeeb5d5727034c1a290, NAME => 'TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:02:11,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2efe576d1bb7feeeb5d5727034c1a290, NAME => 'TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2efe576d1bb7feeeb5d5727034c1a290, NAME => 'TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:02:11,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-20T09:02:11,308 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-20T09:02:11,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290. 2024-11-20T09:02:11,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290. as already flushing 2024-11-20T09:02:11,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290. 2024-11-20T09:02:11,309 ERROR [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 2efe576d1bb7feeeb5d5727034c1a290, NAME => 'TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:02:11,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 2efe576d1bb7feeeb5d5727034c1a290, NAME => 'TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:02:11,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2efe576d1bb7feeeb5d5727034c1a290, NAME => 'TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2efe576d1bb7feeeb5d5727034c1a290, NAME => 'TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T09:02:11,437 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=250 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/.tmp/C/148b1d5802434405a079fe65f4ef5cb4 2024-11-20T09:02:11,441 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/.tmp/A/242ffce3ff2346b0af2cf7c7392478d0 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/A/242ffce3ff2346b0af2cf7c7392478d0 2024-11-20T09:02:11,444 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/A/242ffce3ff2346b0af2cf7c7392478d0, entries=150, sequenceid=250, filesize=30.4 K 2024-11-20T09:02:11,445 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/.tmp/B/53b81d6b3e454d849ba4fe8c81a9021e as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/B/53b81d6b3e454d849ba4fe8c81a9021e 2024-11-20T09:02:11,448 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/B/53b81d6b3e454d849ba4fe8c81a9021e, entries=150, sequenceid=250, filesize=11.9 K 2024-11-20T09:02:11,448 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/.tmp/C/148b1d5802434405a079fe65f4ef5cb4 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/C/148b1d5802434405a079fe65f4ef5cb4 2024-11-20T09:02:11,451 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/C/148b1d5802434405a079fe65f4ef5cb4, entries=150, sequenceid=250, filesize=11.9 K 2024-11-20T09:02:11,452 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 2efe576d1bb7feeeb5d5727034c1a290 in 1662ms, sequenceid=250, compaction requested=true 2024-11-20T09:02:11,452 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2efe576d1bb7feeeb5d5727034c1a290: 2024-11-20T09:02:11,452 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T09:02:11,453 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2efe576d1bb7feeeb5d5727034c1a290:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T09:02:11,453 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T09:02:11,453 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2efe576d1bb7feeeb5d5727034c1a290:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T09:02:11,453 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T09:02:11,453 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2efe576d1bb7feeeb5d5727034c1a290:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T09:02:11,453 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T09:02:11,453 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T09:02:11,457 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49048 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T09:02:11,457 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 133508 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T09:02:11,457 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HStore(1541): 2efe576d1bb7feeeb5d5727034c1a290/B is initiating minor compaction (all files) 2024-11-20T09:02:11,457 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1541): 2efe576d1bb7feeeb5d5727034c1a290/A is initiating minor compaction (all files) 2024-11-20T09:02:11,457 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2efe576d1bb7feeeb5d5727034c1a290/B in TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290. 2024-11-20T09:02:11,457 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2efe576d1bb7feeeb5d5727034c1a290/A in TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290. 2024-11-20T09:02:11,457 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/B/082f3d32b8fa48ba90c235e5e2034313, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/B/0f0f6b0213584768b430eb41475e8182, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/B/a58f882ce9c544a3afb1e3f43869c330, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/B/53b81d6b3e454d849ba4fe8c81a9021e] into tmpdir=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/.tmp, totalSize=47.9 K 2024-11-20T09:02:11,457 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/A/323f78dd4aa2450b8f92bb0d208bc874, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/A/70a78ff38a8e4f509ca1ca1468cd5980, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/A/6850b05bd9bd49bca3bdcc0de66fcd9a, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/A/242ffce3ff2346b0af2cf7c7392478d0] into tmpdir=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/.tmp, totalSize=130.4 K 2024-11-20T09:02:11,457 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290. 2024-11-20T09:02:11,457 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290. files: [hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/A/323f78dd4aa2450b8f92bb0d208bc874, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/A/70a78ff38a8e4f509ca1ca1468cd5980, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/A/6850b05bd9bd49bca3bdcc0de66fcd9a, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/A/242ffce3ff2346b0af2cf7c7392478d0] 2024-11-20T09:02:11,457 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.Compactor(225): Compacting 082f3d32b8fa48ba90c235e5e2034313, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1732093323541 2024-11-20T09:02:11,457 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting 323f78dd4aa2450b8f92bb0d208bc874, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1732093323541 2024-11-20T09:02:11,458 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0f0f6b0213584768b430eb41475e8182, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1732093326358 2024-11-20T09:02:11,458 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting 70a78ff38a8e4f509ca1ca1468cd5980, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1732093325729 2024-11-20T09:02:11,458 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.Compactor(225): Compacting a58f882ce9c544a3afb1e3f43869c330, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1732093326437 2024-11-20T09:02:11,458 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting 6850b05bd9bd49bca3bdcc0de66fcd9a, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1732093326437 2024-11-20T09:02:11,458 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] compactions.Compactor(225): Compacting 53b81d6b3e454d849ba4fe8c81a9021e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1732093328642 2024-11-20T09:02:11,458 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting 242ffce3ff2346b0af2cf7c7392478d0, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1732093328642 2024-11-20T09:02:11,461 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34223 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-20T09:02:11,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290. 2024-11-20T09:02:11,462 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2902): Flushing 2efe576d1bb7feeeb5d5727034c1a290 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-20T09:02:11,466 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=2efe576d1bb7feeeb5d5727034c1a290] 2024-11-20T09:02:11,468 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241120ebabf76f75ae456f856203d117c300f7_2efe576d1bb7feeeb5d5727034c1a290 store=[table=TestAcidGuarantees family=A region=2efe576d1bb7feeeb5d5727034c1a290] 2024-11-20T09:02:11,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112081eeb20477be4177b61af919ddf2f9ec_2efe576d1bb7feeeb5d5727034c1a290 is 50, key is test_row_0/A:col10/1732093329868/Put/seqid=0 2024-11-20T09:02:11,471 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241120ebabf76f75ae456f856203d117c300f7_2efe576d1bb7feeeb5d5727034c1a290, store=[table=TestAcidGuarantees family=A region=2efe576d1bb7feeeb5d5727034c1a290] 2024-11-20T09:02:11,471 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120ebabf76f75ae456f856203d117c300f7_2efe576d1bb7feeeb5d5727034c1a290 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=2efe576d1bb7feeeb5d5727034c1a290] 2024-11-20T09:02:11,471 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2efe576d1bb7feeeb5d5727034c1a290#B#compaction#532 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-20T09:02:11,472 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/.tmp/B/3ea0786197f44013ac5903e2b595bb11 is 50, key is test_row_0/B:col10/1732093328646/Put/seqid=0 2024-11-20T09:02:11,473 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742452_1628 (size=12454) 2024-11-20T09:02:11,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:11,477 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112081eeb20477be4177b61af919ddf2f9ec_2efe576d1bb7feeeb5d5727034c1a290 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112081eeb20477be4177b61af919ddf2f9ec_2efe576d1bb7feeeb5d5727034c1a290 2024-11-20T09:02:11,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-20T09:02:11,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/.tmp/A/30c33c47a49b4ff5a11648471a719591, store: [table=TestAcidGuarantees family=A region=2efe576d1bb7feeeb5d5727034c1a290] 2024-11-20T09:02:11,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/.tmp/A/30c33c47a49b4ff5a11648471a719591 is 175, key is test_row_0/A:col10/1732093329868/Put/seqid=0 2024-11-20T09:02:11,493 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742454_1630 (size=12731) 2024-11-20T09:02:11,498 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742455_1631 (size=31255) 2024-11-20T09:02:11,501 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742453_1629 (size=4469) 2024-11-20T09:02:11,502 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2efe576d1bb7feeeb5d5727034c1a290#A#compaction#530 average throughput is 0.68 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T09:02:11,503 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/.tmp/A/dd244da8bc084b9fa57664b0ea00306f is 175, key is test_row_0/A:col10/1732093328646/Put/seqid=0 2024-11-20T09:02:11,506 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742456_1632 (size=31685) 2024-11-20T09:02:11,511 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/.tmp/A/dd244da8bc084b9fa57664b0ea00306f as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/A/dd244da8bc084b9fa57664b0ea00306f 2024-11-20T09:02:11,515 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 2efe576d1bb7feeeb5d5727034c1a290/A of 2efe576d1bb7feeeb5d5727034c1a290 into dd244da8bc084b9fa57664b0ea00306f(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T09:02:11,515 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2efe576d1bb7feeeb5d5727034c1a290: 2024-11-20T09:02:11,515 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., storeName=2efe576d1bb7feeeb5d5727034c1a290/A, priority=12, startTime=1732093331452; duration=0sec 2024-11-20T09:02:11,515 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T09:02:11,515 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2efe576d1bb7feeeb5d5727034c1a290:A 2024-11-20T09:02:11,515 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T09:02:11,516 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49048 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T09:02:11,516 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1541): 2efe576d1bb7feeeb5d5727034c1a290/C is initiating minor compaction (all files) 2024-11-20T09:02:11,516 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2efe576d1bb7feeeb5d5727034c1a290/C in TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290. 2024-11-20T09:02:11,516 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/C/11b8f70b90d745fba44537b31120d560, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/C/e521a000085e40e9a7574910a2b42c19, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/C/2a745a7663014d95a3952a396a494ba8, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/C/148b1d5802434405a079fe65f4ef5cb4] into tmpdir=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/.tmp, totalSize=47.9 K 2024-11-20T09:02:11,516 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting 11b8f70b90d745fba44537b31120d560, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1732093323541 2024-11-20T09:02:11,517 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting e521a000085e40e9a7574910a2b42c19, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1732093326358 2024-11-20T09:02:11,517 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting 2a745a7663014d95a3952a396a494ba8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1732093326437 2024-11-20T09:02:11,517 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] compactions.Compactor(225): Compacting 148b1d5802434405a079fe65f4ef5cb4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1732093328642 2024-11-20T09:02:11,524 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2efe576d1bb7feeeb5d5727034c1a290#C#compaction#533 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T09:02:11,525 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/.tmp/C/d09cb0e3d3a846378c28a35da70c3ba0 is 50, key is test_row_0/C:col10/1732093328646/Put/seqid=0 2024-11-20T09:02:11,528 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742457_1633 (size=12731) 2024-11-20T09:02:11,899 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=274, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/.tmp/A/30c33c47a49b4ff5a11648471a719591 2024-11-20T09:02:11,905 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/.tmp/B/3ea0786197f44013ac5903e2b595bb11 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/B/3ea0786197f44013ac5903e2b595bb11 2024-11-20T09:02:11,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/.tmp/B/2cfd2d0c5f1f44efb7dd618500c66b77 is 50, key is test_row_0/B:col10/1732093329868/Put/seqid=0 2024-11-20T09:02:11,909 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 2efe576d1bb7feeeb5d5727034c1a290/B of 2efe576d1bb7feeeb5d5727034c1a290 into 3ea0786197f44013ac5903e2b595bb11(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T09:02:11,909 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2efe576d1bb7feeeb5d5727034c1a290: 2024-11-20T09:02:11,909 INFO [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., storeName=2efe576d1bb7feeeb5d5727034c1a290/B, priority=12, startTime=1732093331453; duration=0sec 2024-11-20T09:02:11,910 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T09:02:11,910 DEBUG [RS:0;be1eb2620e0e:34223-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2efe576d1bb7feeeb5d5727034c1a290:B 2024-11-20T09:02:11,911 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742458_1634 (size=12301) 2024-11-20T09:02:11,912 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=274 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/.tmp/B/2cfd2d0c5f1f44efb7dd618500c66b77 2024-11-20T09:02:11,913 INFO [Thread-2496 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2496. 2024-11-20T09:02:11,913 INFO [Thread-2504 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2504. 2024-11-20T09:02:11,913 DEBUG [Thread-2496 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T09:02:11,913 DEBUG [Thread-2504 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T09:02:11,913 DEBUG [Thread-2504 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T09:02:11,913 DEBUG [Thread-2496 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T09:02:11,913 DEBUG [Thread-2504 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T09:02:11,913 DEBUG [Thread-2496 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T09:02:11,913 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T09:02:11,913 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T09:02:11,915 INFO [Thread-2502 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2502. 2024-11-20T09:02:11,915 DEBUG [Thread-2502 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T09:02:11,915 DEBUG [Thread-2502 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T09:02:11,915 DEBUG [Thread-2502 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T09:02:11,915 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T09:02:11,916 INFO [Thread-2500 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2500. 2024-11-20T09:02:11,916 DEBUG [Thread-2500 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T09:02:11,916 DEBUG [Thread-2500 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T09:02:11,916 INFO [Thread-2498 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2498. 2024-11-20T09:02:11,916 DEBUG [Thread-2500 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T09:02:11,916 DEBUG [Thread-2498 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T09:02:11,916 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T09:02:11,916 DEBUG [Thread-2498 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T09:02:11,916 DEBUG [Thread-2498 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T09:02:11,916 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T09:02:11,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/.tmp/C/0a0e03dc723643c3bec3be1cfb6aff1d is 50, key is test_row_0/C:col10/1732093329868/Put/seqid=0 2024-11-20T09:02:11,920 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742459_1635 (size=12301) 2024-11-20T09:02:11,931 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/.tmp/C/d09cb0e3d3a846378c28a35da70c3ba0 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/C/d09cb0e3d3a846378c28a35da70c3ba0 2024-11-20T09:02:11,934 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 2efe576d1bb7feeeb5d5727034c1a290/C of 2efe576d1bb7feeeb5d5727034c1a290 into d09cb0e3d3a846378c28a35da70c3ba0(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T09:02:11,934 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2efe576d1bb7feeeb5d5727034c1a290: 2024-11-20T09:02:11,934 INFO [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., storeName=2efe576d1bb7feeeb5d5727034c1a290/C, priority=12, startTime=1732093331453; duration=0sec 2024-11-20T09:02:11,934 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T09:02:11,934 DEBUG [RS:0;be1eb2620e0e:34223-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2efe576d1bb7feeeb5d5727034c1a290:C 2024-11-20T09:02:11,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-20T09:02:12,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34223 {}] regionserver.HRegion(8855): Flush requested on 2efe576d1bb7feeeb5d5727034c1a290 2024-11-20T09:02:12,038 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290. as already flushing 2024-11-20T09:02:12,039 INFO [Thread-2487 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2487. 2024-11-20T09:02:12,039 DEBUG [Thread-2487 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T09:02:12,039 DEBUG [Thread-2487 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T09:02:12,039 DEBUG [Thread-2487 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T09:02:12,039 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T09:02:12,321 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=274 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/.tmp/C/0a0e03dc723643c3bec3be1cfb6aff1d 2024-11-20T09:02:12,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/.tmp/A/30c33c47a49b4ff5a11648471a719591 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/A/30c33c47a49b4ff5a11648471a719591 2024-11-20T09:02:12,327 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/A/30c33c47a49b4ff5a11648471a719591, entries=150, sequenceid=274, filesize=30.5 K 2024-11-20T09:02:12,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/.tmp/B/2cfd2d0c5f1f44efb7dd618500c66b77 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/B/2cfd2d0c5f1f44efb7dd618500c66b77 2024-11-20T09:02:12,330 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/B/2cfd2d0c5f1f44efb7dd618500c66b77, entries=150, sequenceid=274, filesize=12.0 K 2024-11-20T09:02:12,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/.tmp/C/0a0e03dc723643c3bec3be1cfb6aff1d as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/C/0a0e03dc723643c3bec3be1cfb6aff1d 2024-11-20T09:02:12,333 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/C/0a0e03dc723643c3bec3be1cfb6aff1d, entries=150, sequenceid=274, filesize=12.0 K 2024-11-20T09:02:12,334 INFO [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=6.71 KB/6870 for 2efe576d1bb7feeeb5d5727034c1a290 in 871ms, sequenceid=274, compaction requested=false 2024-11-20T09:02:12,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2603): Flush status journal for 2efe576d1bb7feeeb5d5727034c1a290: 2024-11-20T09:02:12,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290. 2024-11-20T09:02:12,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/be1eb2620e0e:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=157 2024-11-20T09:02:12,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.HMaster(4169): Remote procedure done, pid=157 2024-11-20T09:02:12,336 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=157, resume processing ppid=156 2024-11-20T09:02:12,336 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=157, ppid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4850 sec 2024-11-20T09:02:12,337 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees in 1.4880 sec 2024-11-20T09:02:12,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-20T09:02:12,997 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T09:02:14,638 INFO [Thread-2489 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2489. 2024-11-20T09:02:14,638 DEBUG [Thread-2489 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T09:02:14,638 DEBUG [Thread-2489 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T09:02:14,638 DEBUG [Thread-2489 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T09:02:14,638 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T09:02:14,658 INFO [Thread-2485 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2485. 2024-11-20T09:02:14,658 DEBUG [Thread-2485 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T09:02:14,658 DEBUG [Thread-2485 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T09:02:14,658 DEBUG [Thread-2485 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T09:02:14,658 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T09:02:14,668 INFO [Thread-2493 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2493. 2024-11-20T09:02:14,668 DEBUG [Thread-2493 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T09:02:14,668 DEBUG [Thread-2493 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T09:02:14,668 DEBUG [Thread-2493 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T09:02:14,668 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T09:02:14,689 INFO [Thread-2491 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2491. 2024-11-20T09:02:14,689 DEBUG [Thread-2491 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T09:02:14,689 DEBUG [Thread-2491 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T09:02:14,689 DEBUG [Thread-2491 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T09:02:14,689 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T09:02:14,689 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-20T09:02:14,689 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 15 2024-11-20T09:02:14,689 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 158 2024-11-20T09:02:14,689 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 16 2024-11-20T09:02:14,689 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 16 2024-11-20T09:02:14,689 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 16 2024-11-20T09:02:14,689 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-20T09:02:14,689 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3674 2024-11-20T09:02:14,689 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3501 2024-11-20T09:02:14,689 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3395 2024-11-20T09:02:14,689 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3659 2024-11-20T09:02:14,689 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3497 2024-11-20T09:02:14,689 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-20T09:02:14,689 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-20T09:02:14,689 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobGetAtomicity(AcidGuaranteesTestBase.java:122) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-20T09:02:14,689 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T09:02:14,689 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T09:02:14,690 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T09:02:14,690 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-20T09:02:14,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] procedure2.ProcedureExecutor(1139): Stored pid=158, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-20T09:02:14,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-20T09:02:14,694 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732093334693"}]},"ts":"1732093334693"} 2024-11-20T09:02:14,695 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-20T09:02:14,695 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-20T09:02:14,695 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=159, ppid=158, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-20T09:02:14,696 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=160, ppid=159, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2efe576d1bb7feeeb5d5727034c1a290, UNASSIGN}] 2024-11-20T09:02:14,697 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=160, ppid=159, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2efe576d1bb7feeeb5d5727034c1a290, UNASSIGN 2024-11-20T09:02:14,697 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=160 updating hbase:meta row=2efe576d1bb7feeeb5d5727034c1a290, regionState=CLOSING, regionLocation=be1eb2620e0e,34223,1732093159960 2024-11-20T09:02:14,698 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=160, ppid=159, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=2efe576d1bb7feeeb5d5727034c1a290, UNASSIGN because future has completed 2024-11-20T09:02:14,699 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-20T09:02:14,699 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=161, ppid=160, state=RUNNABLE, hasLock=false; CloseRegionProcedure 2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960}] 2024-11-20T09:02:14,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-20T09:02:14,851 INFO [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] handler.UnassignRegionHandler(122): Close 2efe576d1bb7feeeb5d5727034c1a290 2024-11-20T09:02:14,851 DEBUG [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-20T09:02:14,851 DEBUG [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.HRegion(1722): Closing 2efe576d1bb7feeeb5d5727034c1a290, disabling compactions & flushes 2024-11-20T09:02:14,851 INFO [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290. 2024-11-20T09:02:14,851 DEBUG [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290. 2024-11-20T09:02:14,851 DEBUG [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290. after waiting 0 ms 2024-11-20T09:02:14,851 DEBUG [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290. 2024-11-20T09:02:14,851 INFO [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.HRegion(2902): Flushing 2efe576d1bb7feeeb5d5727034c1a290 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-20T09:02:14,856 DEBUG [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411204487caa05c5344dda06eb0491c31ab01_2efe576d1bb7feeeb5d5727034c1a290 is 50, key is test_row_0/A:col10/1732093334687/Put/seqid=0 2024-11-20T09:02:14,858 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742460_1636 (size=9914) 2024-11-20T09:02:15,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-20T09:02:15,259 DEBUG [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T09:02:15,262 INFO [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411204487caa05c5344dda06eb0491c31ab01_2efe576d1bb7feeeb5d5727034c1a290 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411204487caa05c5344dda06eb0491c31ab01_2efe576d1bb7feeeb5d5727034c1a290 2024-11-20T09:02:15,262 DEBUG [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/.tmp/A/68e7863a542b41948b9fa2e64da702e0, store: [table=TestAcidGuarantees family=A region=2efe576d1bb7feeeb5d5727034c1a290] 2024-11-20T09:02:15,263 DEBUG [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/.tmp/A/68e7863a542b41948b9fa2e64da702e0 is 175, key is test_row_0/A:col10/1732093334687/Put/seqid=0 2024-11-20T09:02:15,266 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742461_1637 (size=22561) 2024-11-20T09:02:15,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-20T09:02:15,666 INFO [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=285, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/.tmp/A/68e7863a542b41948b9fa2e64da702e0 2024-11-20T09:02:15,671 DEBUG [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/.tmp/B/6f935ad3f4354277b6d925de44e927dc is 50, key is test_row_0/B:col10/1732093334687/Put/seqid=0 2024-11-20T09:02:15,674 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742462_1638 (size=9857) 2024-11-20T09:02:15,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-20T09:02:16,075 INFO [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=285 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/.tmp/B/6f935ad3f4354277b6d925de44e927dc 2024-11-20T09:02:16,080 DEBUG [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/.tmp/C/223868fdb915402ab54ca0538aadb374 is 50, key is test_row_0/C:col10/1732093334687/Put/seqid=0 2024-11-20T09:02:16,082 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742463_1639 (size=9857) 2024-11-20T09:02:16,483 INFO [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=285 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/.tmp/C/223868fdb915402ab54ca0538aadb374 2024-11-20T09:02:16,486 DEBUG [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/.tmp/A/68e7863a542b41948b9fa2e64da702e0 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/A/68e7863a542b41948b9fa2e64da702e0 2024-11-20T09:02:16,489 INFO [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/A/68e7863a542b41948b9fa2e64da702e0, entries=100, sequenceid=285, filesize=22.0 K 2024-11-20T09:02:16,490 DEBUG [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/.tmp/B/6f935ad3f4354277b6d925de44e927dc as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/B/6f935ad3f4354277b6d925de44e927dc 2024-11-20T09:02:16,492 INFO [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/B/6f935ad3f4354277b6d925de44e927dc, entries=100, sequenceid=285, filesize=9.6 K 2024-11-20T09:02:16,493 DEBUG [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/.tmp/C/223868fdb915402ab54ca0538aadb374 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/C/223868fdb915402ab54ca0538aadb374 2024-11-20T09:02:16,495 INFO [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/C/223868fdb915402ab54ca0538aadb374, entries=100, sequenceid=285, filesize=9.6 K 2024-11-20T09:02:16,495 INFO [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 2efe576d1bb7feeeb5d5727034c1a290 in 1644ms, sequenceid=285, compaction requested=true 2024-11-20T09:02:16,496 DEBUG [StoreCloser-TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/A/204bd333446043fa8a8c9beb08a5f232, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/A/737b2ab810cd4f8d887134abda2f5634, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/A/763115a23ee04e84a31acd8f405995e7, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/A/7c9f0c17f70b4264bf90e44a3c33d8f8, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/A/88a6fdd2c5f14f78a80a75e3bffb79f8, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/A/155c144420354405bee0048adb1006dc, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/A/f288fe87ca7141bf99732dfa7941b872, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/A/45b817b98eec4a1c85086a07ac8101e9, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/A/056d297ddd93429fbc52fefff1e2fee0, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/A/39631abd030a4b0389e9a5a04a1234e6, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/A/097bb2dc8c6943b2bce068d498b839bd, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/A/331981096486447898ee96d3c3d23a2a, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/A/323f78dd4aa2450b8f92bb0d208bc874, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/A/8bcffbd0558c4204b995c949a27bed12, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/A/70a78ff38a8e4f509ca1ca1468cd5980, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/A/6850b05bd9bd49bca3bdcc0de66fcd9a, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/A/242ffce3ff2346b0af2cf7c7392478d0] to archive 2024-11-20T09:02:16,497 DEBUG [StoreCloser-TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-20T09:02:16,498 DEBUG [StoreCloser-TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/A/204bd333446043fa8a8c9beb08a5f232 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/A/204bd333446043fa8a8c9beb08a5f232 2024-11-20T09:02:16,499 DEBUG [StoreCloser-TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/A/737b2ab810cd4f8d887134abda2f5634 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/A/737b2ab810cd4f8d887134abda2f5634 2024-11-20T09:02:16,500 DEBUG [StoreCloser-TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/A/763115a23ee04e84a31acd8f405995e7 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/A/763115a23ee04e84a31acd8f405995e7 2024-11-20T09:02:16,501 DEBUG [StoreCloser-TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/A/7c9f0c17f70b4264bf90e44a3c33d8f8 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/A/7c9f0c17f70b4264bf90e44a3c33d8f8 2024-11-20T09:02:16,502 DEBUG [StoreCloser-TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/A/88a6fdd2c5f14f78a80a75e3bffb79f8 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/A/88a6fdd2c5f14f78a80a75e3bffb79f8 2024-11-20T09:02:16,502 DEBUG [StoreCloser-TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/A/155c144420354405bee0048adb1006dc to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/A/155c144420354405bee0048adb1006dc 2024-11-20T09:02:16,503 DEBUG [StoreCloser-TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/A/f288fe87ca7141bf99732dfa7941b872 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/A/f288fe87ca7141bf99732dfa7941b872 2024-11-20T09:02:16,504 DEBUG [StoreCloser-TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/A/45b817b98eec4a1c85086a07ac8101e9 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/A/45b817b98eec4a1c85086a07ac8101e9 2024-11-20T09:02:16,505 DEBUG [StoreCloser-TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/A/056d297ddd93429fbc52fefff1e2fee0 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/A/056d297ddd93429fbc52fefff1e2fee0 2024-11-20T09:02:16,506 DEBUG [StoreCloser-TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/A/39631abd030a4b0389e9a5a04a1234e6 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/A/39631abd030a4b0389e9a5a04a1234e6 2024-11-20T09:02:16,507 DEBUG [StoreCloser-TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/A/097bb2dc8c6943b2bce068d498b839bd to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/A/097bb2dc8c6943b2bce068d498b839bd 2024-11-20T09:02:16,507 DEBUG [StoreCloser-TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/A/331981096486447898ee96d3c3d23a2a to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/A/331981096486447898ee96d3c3d23a2a 2024-11-20T09:02:16,508 DEBUG [StoreCloser-TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/A/323f78dd4aa2450b8f92bb0d208bc874 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/A/323f78dd4aa2450b8f92bb0d208bc874 2024-11-20T09:02:16,509 DEBUG [StoreCloser-TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/A/8bcffbd0558c4204b995c949a27bed12 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/A/8bcffbd0558c4204b995c949a27bed12 2024-11-20T09:02:16,510 DEBUG [StoreCloser-TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/A/70a78ff38a8e4f509ca1ca1468cd5980 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/A/70a78ff38a8e4f509ca1ca1468cd5980 2024-11-20T09:02:16,511 DEBUG [StoreCloser-TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/A/6850b05bd9bd49bca3bdcc0de66fcd9a to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/A/6850b05bd9bd49bca3bdcc0de66fcd9a 2024-11-20T09:02:16,511 DEBUG [StoreCloser-TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/A/242ffce3ff2346b0af2cf7c7392478d0 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/A/242ffce3ff2346b0af2cf7c7392478d0 2024-11-20T09:02:16,512 DEBUG [StoreCloser-TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/B/919cbddd2375495abf2a7ae69c21d872, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/B/cdd4ea03fc7b4a1e8cd7e4d5d2a62046, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/B/7e9e4b7ab89e4728af657370090f5bee, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/B/e1646054bd104f1c962cf971cdf08c3c, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/B/8c9f66c0eef347d4b0fb682f5b5a90ac, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/B/188e0c11698f458a86aa3d4de45b0f31, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/B/9ca1284dbf6842f1a7001bfe7ba44eeb, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/B/ace8ca1ee71547e8b283eee4a8b7e6a0, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/B/544630a02cdf4b11a238078ccee09bf1, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/B/258b6dbd94b94f218dca88b140d529f8, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/B/ff0108518d3f4698a41614145284afb9, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/B/0e93702288784452abdf98d200cdb5c1, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/B/082f3d32b8fa48ba90c235e5e2034313, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/B/e13fd36033ce44ff8a864dbbf0ed2b5b, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/B/0f0f6b0213584768b430eb41475e8182, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/B/a58f882ce9c544a3afb1e3f43869c330, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/B/53b81d6b3e454d849ba4fe8c81a9021e] to archive 2024-11-20T09:02:16,513 DEBUG [StoreCloser-TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-20T09:02:16,514 DEBUG [StoreCloser-TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/B/919cbddd2375495abf2a7ae69c21d872 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/B/919cbddd2375495abf2a7ae69c21d872 2024-11-20T09:02:16,515 DEBUG [StoreCloser-TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/B/cdd4ea03fc7b4a1e8cd7e4d5d2a62046 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/B/cdd4ea03fc7b4a1e8cd7e4d5d2a62046 2024-11-20T09:02:16,516 DEBUG [StoreCloser-TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/B/7e9e4b7ab89e4728af657370090f5bee to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/B/7e9e4b7ab89e4728af657370090f5bee 2024-11-20T09:02:16,517 DEBUG [StoreCloser-TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/B/e1646054bd104f1c962cf971cdf08c3c to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/B/e1646054bd104f1c962cf971cdf08c3c 2024-11-20T09:02:16,518 DEBUG [StoreCloser-TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/B/8c9f66c0eef347d4b0fb682f5b5a90ac to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/B/8c9f66c0eef347d4b0fb682f5b5a90ac 2024-11-20T09:02:16,519 DEBUG [StoreCloser-TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/B/188e0c11698f458a86aa3d4de45b0f31 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/B/188e0c11698f458a86aa3d4de45b0f31 2024-11-20T09:02:16,520 DEBUG [StoreCloser-TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/B/9ca1284dbf6842f1a7001bfe7ba44eeb to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/B/9ca1284dbf6842f1a7001bfe7ba44eeb 2024-11-20T09:02:16,521 DEBUG [StoreCloser-TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/B/ace8ca1ee71547e8b283eee4a8b7e6a0 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/B/ace8ca1ee71547e8b283eee4a8b7e6a0 2024-11-20T09:02:16,521 DEBUG [StoreCloser-TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/B/544630a02cdf4b11a238078ccee09bf1 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/B/544630a02cdf4b11a238078ccee09bf1 2024-11-20T09:02:16,522 DEBUG [StoreCloser-TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/B/258b6dbd94b94f218dca88b140d529f8 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/B/258b6dbd94b94f218dca88b140d529f8 2024-11-20T09:02:16,523 DEBUG [StoreCloser-TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/B/ff0108518d3f4698a41614145284afb9 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/B/ff0108518d3f4698a41614145284afb9 2024-11-20T09:02:16,524 DEBUG [StoreCloser-TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/B/0e93702288784452abdf98d200cdb5c1 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/B/0e93702288784452abdf98d200cdb5c1 2024-11-20T09:02:16,524 DEBUG [StoreCloser-TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/B/082f3d32b8fa48ba90c235e5e2034313 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/B/082f3d32b8fa48ba90c235e5e2034313 2024-11-20T09:02:16,525 DEBUG [StoreCloser-TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/B/e13fd36033ce44ff8a864dbbf0ed2b5b to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/B/e13fd36033ce44ff8a864dbbf0ed2b5b 2024-11-20T09:02:16,526 DEBUG [StoreCloser-TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/B/0f0f6b0213584768b430eb41475e8182 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/B/0f0f6b0213584768b430eb41475e8182 2024-11-20T09:02:16,527 DEBUG [StoreCloser-TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/B/a58f882ce9c544a3afb1e3f43869c330 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/B/a58f882ce9c544a3afb1e3f43869c330 2024-11-20T09:02:16,527 DEBUG [StoreCloser-TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/B/53b81d6b3e454d849ba4fe8c81a9021e to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/B/53b81d6b3e454d849ba4fe8c81a9021e 2024-11-20T09:02:16,529 DEBUG [StoreCloser-TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/C/bfd37f541ac94c37a67737eca6a42199, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/C/22dac69b05d8410a9ca31ddc8782e016, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/C/6a6388e84b7b47f38ebe2f81cec5a14f, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/C/beba8a8f994d40daa025c3e35b40d25c, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/C/6aadc71ddde848bd85515c14c179107b, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/C/848c0c5a29ad4244ac07408b9cf158f7, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/C/f118f2f791014323ba76947729c2bf22, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/C/f9c80e049fe34754bc930e8430edc6c9, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/C/4df61a541594426a8a3d5dbe15cf7647, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/C/bc89334ca1e245e0a7d498e1ba63e8b9, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/C/054ab2d467da47b48610e6b11130a4d4, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/C/f2d0e864812f432f855b5de552fc6cc4, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/C/11b8f70b90d745fba44537b31120d560, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/C/f1cd2c657db84a638ce6f74ac6471683, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/C/e521a000085e40e9a7574910a2b42c19, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/C/2a745a7663014d95a3952a396a494ba8, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/C/148b1d5802434405a079fe65f4ef5cb4] to archive 2024-11-20T09:02:16,529 DEBUG [StoreCloser-TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-20T09:02:16,530 DEBUG [StoreCloser-TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/C/bfd37f541ac94c37a67737eca6a42199 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/C/bfd37f541ac94c37a67737eca6a42199 2024-11-20T09:02:16,531 DEBUG [StoreCloser-TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/C/22dac69b05d8410a9ca31ddc8782e016 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/C/22dac69b05d8410a9ca31ddc8782e016 2024-11-20T09:02:16,532 DEBUG [StoreCloser-TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/C/6a6388e84b7b47f38ebe2f81cec5a14f to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/C/6a6388e84b7b47f38ebe2f81cec5a14f 2024-11-20T09:02:16,532 DEBUG [StoreCloser-TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/C/beba8a8f994d40daa025c3e35b40d25c to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/C/beba8a8f994d40daa025c3e35b40d25c 2024-11-20T09:02:16,533 DEBUG [StoreCloser-TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/C/6aadc71ddde848bd85515c14c179107b to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/C/6aadc71ddde848bd85515c14c179107b 2024-11-20T09:02:16,534 DEBUG [StoreCloser-TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/C/848c0c5a29ad4244ac07408b9cf158f7 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/C/848c0c5a29ad4244ac07408b9cf158f7 2024-11-20T09:02:16,534 DEBUG [StoreCloser-TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/C/f118f2f791014323ba76947729c2bf22 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/C/f118f2f791014323ba76947729c2bf22 2024-11-20T09:02:16,535 DEBUG [StoreCloser-TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/C/f9c80e049fe34754bc930e8430edc6c9 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/C/f9c80e049fe34754bc930e8430edc6c9 2024-11-20T09:02:16,536 DEBUG [StoreCloser-TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/C/4df61a541594426a8a3d5dbe15cf7647 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/C/4df61a541594426a8a3d5dbe15cf7647 2024-11-20T09:02:16,537 DEBUG [StoreCloser-TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/C/bc89334ca1e245e0a7d498e1ba63e8b9 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/C/bc89334ca1e245e0a7d498e1ba63e8b9 2024-11-20T09:02:16,537 DEBUG [StoreCloser-TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/C/054ab2d467da47b48610e6b11130a4d4 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/C/054ab2d467da47b48610e6b11130a4d4 2024-11-20T09:02:16,538 DEBUG [StoreCloser-TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/C/f2d0e864812f432f855b5de552fc6cc4 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/C/f2d0e864812f432f855b5de552fc6cc4 2024-11-20T09:02:16,539 DEBUG [StoreCloser-TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/C/11b8f70b90d745fba44537b31120d560 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/C/11b8f70b90d745fba44537b31120d560 2024-11-20T09:02:16,539 DEBUG [StoreCloser-TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/C/f1cd2c657db84a638ce6f74ac6471683 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/C/f1cd2c657db84a638ce6f74ac6471683 2024-11-20T09:02:16,540 DEBUG [StoreCloser-TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/C/e521a000085e40e9a7574910a2b42c19 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/C/e521a000085e40e9a7574910a2b42c19 2024-11-20T09:02:16,541 DEBUG [StoreCloser-TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/C/2a745a7663014d95a3952a396a494ba8 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/C/2a745a7663014d95a3952a396a494ba8 2024-11-20T09:02:16,541 DEBUG [StoreCloser-TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/C/148b1d5802434405a079fe65f4ef5cb4 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/C/148b1d5802434405a079fe65f4ef5cb4 2024-11-20T09:02:16,554 DEBUG [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/recovered.edits/288.seqid, newMaxSeqId=288, maxSeqId=4 2024-11-20T09:02:16,555 INFO [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290. 2024-11-20T09:02:16,555 DEBUG [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.HRegion(1676): Region close journal for 2efe576d1bb7feeeb5d5727034c1a290: Waiting for close lock at 1732093334851Running coprocessor pre-close hooks at 1732093334851Disabling compacts and flushes for region at 1732093334851Disabling writes for close at 1732093334851Obtaining lock to block concurrent updates at 1732093334851Preparing flush snapshotting stores in 2efe576d1bb7feeeb5d5727034c1a290 at 1732093334851Finished memstore snapshotting TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=90720, getOffHeapSize=0, getCellsCount=750 at 1732093334851Flushing stores of TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290. at 1732093334852 (+1 ms)Flushing 2efe576d1bb7feeeb5d5727034c1a290/A: creating writer at 1732093334852Flushing mob file 2efe576d1bb7feeeb5d5727034c1a290/A: appending metadata at 1732093334855 (+3 ms)Flushing mob file 2efe576d1bb7feeeb5d5727034c1a290/A: closing flushed file at 1732093334855Flushing 2efe576d1bb7feeeb5d5727034c1a290/A: appending metadata at 1732093335262 (+407 ms)Flushing 2efe576d1bb7feeeb5d5727034c1a290/A: closing flushed file at 1732093335263 (+1 ms)Flushing 2efe576d1bb7feeeb5d5727034c1a290/B: creating writer at 1732093335669 (+406 ms)Flushing 2efe576d1bb7feeeb5d5727034c1a290/B: appending metadata at 1732093335671 (+2 ms)Flushing 2efe576d1bb7feeeb5d5727034c1a290/B: closing flushed file at 1732093335671Flushing 2efe576d1bb7feeeb5d5727034c1a290/C: creating writer at 1732093336077 (+406 ms)Flushing 2efe576d1bb7feeeb5d5727034c1a290/C: appending metadata at 1732093336079 (+2 ms)Flushing 2efe576d1bb7feeeb5d5727034c1a290/C: closing flushed file at 1732093336079Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@41ee53a6: reopening flushed file at 1732093336486 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@57d4977e: reopening flushed file at 1732093336489 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5dac78e7: reopening flushed file at 1732093336492 (+3 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 2efe576d1bb7feeeb5d5727034c1a290 in 1644ms, sequenceid=285, compaction requested=true at 1732093336495 (+3 ms)Writing region close event to WAL at 1732093336542 (+47 ms)Running coprocessor post-close hooks at 1732093336555 (+13 ms)Closed at 1732093336555 2024-11-20T09:02:16,557 INFO [RS_CLOSE_REGION-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] handler.UnassignRegionHandler(157): Closed 2efe576d1bb7feeeb5d5727034c1a290 2024-11-20T09:02:16,557 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=160 updating hbase:meta row=2efe576d1bb7feeeb5d5727034c1a290, regionState=CLOSED 2024-11-20T09:02:16,558 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=161, ppid=160, state=RUNNABLE, hasLock=false; CloseRegionProcedure 2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 because future has completed 2024-11-20T09:02:16,560 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=161, resume processing ppid=160 2024-11-20T09:02:16,560 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=161, ppid=160, state=SUCCESS, hasLock=false; CloseRegionProcedure 2efe576d1bb7feeeb5d5727034c1a290, server=be1eb2620e0e,34223,1732093159960 in 1.8600 sec 2024-11-20T09:02:16,561 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=160, resume processing ppid=159 2024-11-20T09:02:16,561 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=160, ppid=159, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2efe576d1bb7feeeb5d5727034c1a290, UNASSIGN in 1.8640 sec 2024-11-20T09:02:16,563 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=159, resume processing ppid=158 2024-11-20T09:02:16,563 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=159, ppid=158, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.8670 sec 2024-11-20T09:02:16,564 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732093336564"}]},"ts":"1732093336564"} 2024-11-20T09:02:16,565 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-20T09:02:16,565 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-20T09:02:16,566 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=158, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.8750 sec 2024-11-20T09:02:16,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-20T09:02:16,837 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-20T09:02:16,838 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-20T09:02:16,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] procedure2.ProcedureExecutor(1139): Stored pid=162, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T09:02:16,839 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=162, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T09:02:16,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-20T09:02:16,840 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=162, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T09:02:16,842 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290 2024-11-20T09:02:16,844 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/A, FileablePath, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/B, FileablePath, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/C, FileablePath, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/recovered.edits] 2024-11-20T09:02:16,845 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/A/30c33c47a49b4ff5a11648471a719591 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/A/30c33c47a49b4ff5a11648471a719591 2024-11-20T09:02:16,846 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/A/68e7863a542b41948b9fa2e64da702e0 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/A/68e7863a542b41948b9fa2e64da702e0 2024-11-20T09:02:16,847 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/A/dd244da8bc084b9fa57664b0ea00306f to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/A/dd244da8bc084b9fa57664b0ea00306f 2024-11-20T09:02:16,848 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/B/2cfd2d0c5f1f44efb7dd618500c66b77 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/B/2cfd2d0c5f1f44efb7dd618500c66b77 2024-11-20T09:02:16,849 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/B/3ea0786197f44013ac5903e2b595bb11 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/B/3ea0786197f44013ac5903e2b595bb11 2024-11-20T09:02:16,850 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/B/6f935ad3f4354277b6d925de44e927dc to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/B/6f935ad3f4354277b6d925de44e927dc 2024-11-20T09:02:16,852 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/C/0a0e03dc723643c3bec3be1cfb6aff1d to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/C/0a0e03dc723643c3bec3be1cfb6aff1d 2024-11-20T09:02:16,853 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/C/223868fdb915402ab54ca0538aadb374 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/C/223868fdb915402ab54ca0538aadb374 2024-11-20T09:02:16,853 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/C/d09cb0e3d3a846378c28a35da70c3ba0 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/C/d09cb0e3d3a846378c28a35da70c3ba0 2024-11-20T09:02:16,855 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/recovered.edits/288.seqid to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290/recovered.edits/288.seqid 2024-11-20T09:02:16,856 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/default/TestAcidGuarantees/2efe576d1bb7feeeb5d5727034c1a290 2024-11-20T09:02:16,856 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-20T09:02:16,856 DEBUG [PEWorker-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-20T09:02:16,857 DEBUG [PEWorker-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-20T09:02:16,859 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112011ff8a7cf7354cecb45b47d16fe50737_2efe576d1bb7feeeb5d5727034c1a290 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112011ff8a7cf7354cecb45b47d16fe50737_2efe576d1bb7feeeb5d5727034c1a290 2024-11-20T09:02:16,860 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112015650b08cc1d4a65a407beb964f58fae_2efe576d1bb7feeeb5d5727034c1a290 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112015650b08cc1d4a65a407beb964f58fae_2efe576d1bb7feeeb5d5727034c1a290 2024-11-20T09:02:16,860 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112026eb151bf6564975a36ee1a5429b4aef_2efe576d1bb7feeeb5d5727034c1a290 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112026eb151bf6564975a36ee1a5429b4aef_2efe576d1bb7feeeb5d5727034c1a290 2024-11-20T09:02:16,861 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112028ed1a493a1d4664967b3475e3f105e7_2efe576d1bb7feeeb5d5727034c1a290 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112028ed1a493a1d4664967b3475e3f105e7_2efe576d1bb7feeeb5d5727034c1a290 2024-11-20T09:02:16,862 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411204487caa05c5344dda06eb0491c31ab01_2efe576d1bb7feeeb5d5727034c1a290 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411204487caa05c5344dda06eb0491c31ab01_2efe576d1bb7feeeb5d5727034c1a290 2024-11-20T09:02:16,863 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411204fdc260e158c4312a01c7eed60f59740_2efe576d1bb7feeeb5d5727034c1a290 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411204fdc260e158c4312a01c7eed60f59740_2efe576d1bb7feeeb5d5727034c1a290 2024-11-20T09:02:16,864 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112060b1a264ae9140d9bc4e841e5f0c6630_2efe576d1bb7feeeb5d5727034c1a290 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112060b1a264ae9140d9bc4e841e5f0c6630_2efe576d1bb7feeeb5d5727034c1a290 2024-11-20T09:02:16,865 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112067c9ca74eebd42beb616958879ccb8e6_2efe576d1bb7feeeb5d5727034c1a290 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112067c9ca74eebd42beb616958879ccb8e6_2efe576d1bb7feeeb5d5727034c1a290 2024-11-20T09:02:16,866 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112081eeb20477be4177b61af919ddf2f9ec_2efe576d1bb7feeeb5d5727034c1a290 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112081eeb20477be4177b61af919ddf2f9ec_2efe576d1bb7feeeb5d5727034c1a290 2024-11-20T09:02:16,866 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411208852ea0b2cb448159ab7b33d76f4f98d_2efe576d1bb7feeeb5d5727034c1a290 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411208852ea0b2cb448159ab7b33d76f4f98d_2efe576d1bb7feeeb5d5727034c1a290 2024-11-20T09:02:16,867 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411208c2f95454851462b8733de46c109417a_2efe576d1bb7feeeb5d5727034c1a290 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411208c2f95454851462b8733de46c109417a_2efe576d1bb7feeeb5d5727034c1a290 2024-11-20T09:02:16,868 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120a1d04945caa34377868db9913d604cce_2efe576d1bb7feeeb5d5727034c1a290 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120a1d04945caa34377868db9913d604cce_2efe576d1bb7feeeb5d5727034c1a290 2024-11-20T09:02:16,869 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120db9e3dc4e1e747f3bd1ab09cab660c29_2efe576d1bb7feeeb5d5727034c1a290 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120db9e3dc4e1e747f3bd1ab09cab660c29_2efe576d1bb7feeeb5d5727034c1a290 2024-11-20T09:02:16,870 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120eca9676d4eea439b8ddf07c5f3dd10ec_2efe576d1bb7feeeb5d5727034c1a290 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120eca9676d4eea439b8ddf07c5f3dd10ec_2efe576d1bb7feeeb5d5727034c1a290 2024-11-20T09:02:16,870 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120efbe038c59774ee4b0f1b4620ddcc8fe_2efe576d1bb7feeeb5d5727034c1a290 to hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120efbe038c59774ee4b0f1b4620ddcc8fe_2efe576d1bb7feeeb5d5727034c1a290 2024-11-20T09:02:16,871 DEBUG [PEWorker-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-20T09:02:16,873 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=162, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T09:02:16,874 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-20T09:02:16,876 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-20T09:02:16,877 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=162, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T09:02:16,877 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-20T09:02:16,877 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732093336877"}]},"ts":"9223372036854775807"} 2024-11-20T09:02:16,878 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-20T09:02:16,878 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 2efe576d1bb7feeeb5d5727034c1a290, NAME => 'TestAcidGuarantees,,1732093308816.2efe576d1bb7feeeb5d5727034c1a290.', STARTKEY => '', ENDKEY => ''}] 2024-11-20T09:02:16,878 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-20T09:02:16,878 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732093336878"}]},"ts":"9223372036854775807"} 2024-11-20T09:02:16,880 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-20T09:02:16,880 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=162, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T09:02:16,881 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=162, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 42 msec 2024-11-20T09:02:16,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44373 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-20T09:02:16,947 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-20T09:02:16,947 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-20T09:02:16,956 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobGetAtomicity Thread=240 (was 242), OpenFileDescriptor=453 (was 455), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=365 (was 384), ProcessCount=11 (was 11), AvailableMemoryMB=1947 (was 1987) 2024-11-20T09:02:16,956 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-20T09:02:16,956 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-20T09:02:16,957 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-20T09:02:16,957 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T09:02:16,957 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T09:02:16,957 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T09:02:16,957 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-20T09:02:16,957 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1143773905, stopped=false 2024-11-20T09:02:16,957 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=be1eb2620e0e,44373,1732093159150 2024-11-20T09:02:16,959 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44373-0x1001427dc570000, quorum=127.0.0.1:52155, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-20T09:02:16,959 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34223-0x1001427dc570001, quorum=127.0.0.1:52155, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-20T09:02:16,959 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34223-0x1001427dc570001, quorum=127.0.0.1:52155, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T09:02:16,959 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44373-0x1001427dc570000, quorum=127.0.0.1:52155, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T09:02:16,959 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T09:02:16,960 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-20T09:02:16,960 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-20T09:02:16,960 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T09:02:16,960 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:44373-0x1001427dc570000, quorum=127.0.0.1:52155, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-20T09:02:16,961 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'be1eb2620e0e,34223,1732093159960' ***** 2024-11-20T09:02:16,961 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-20T09:02:16,961 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:34223-0x1001427dc570001, quorum=127.0.0.1:52155, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-20T09:02:16,961 INFO [RS:0;be1eb2620e0e:34223 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-20T09:02:16,961 INFO [RS:0;be1eb2620e0e:34223 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-20T09:02:16,961 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-20T09:02:16,961 INFO [RS:0;be1eb2620e0e:34223 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-20T09:02:16,961 INFO [RS:0;be1eb2620e0e:34223 {}] regionserver.HRegionServer(959): stopping server be1eb2620e0e,34223,1732093159960 2024-11-20T09:02:16,961 INFO [RS:0;be1eb2620e0e:34223 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-20T09:02:16,962 INFO [RS:0;be1eb2620e0e:34223 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;be1eb2620e0e:34223. 2024-11-20T09:02:16,962 DEBUG [RS:0;be1eb2620e0e:34223 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-20T09:02:16,962 DEBUG [RS:0;be1eb2620e0e:34223 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T09:02:16,962 INFO [RS:0;be1eb2620e0e:34223 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-20T09:02:16,962 INFO [RS:0;be1eb2620e0e:34223 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-20T09:02:16,962 INFO [RS:0;be1eb2620e0e:34223 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-20T09:02:16,962 INFO [RS:0;be1eb2620e0e:34223 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-20T09:02:16,962 INFO [RS:0;be1eb2620e0e:34223 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-20T09:02:16,963 DEBUG [RS:0;be1eb2620e0e:34223 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-20T09:02:16,963 DEBUG [RS:0;be1eb2620e0e:34223 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-20T09:02:16,963 DEBUG [RS_CLOSE_META-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-20T09:02:16,963 INFO [RS_CLOSE_META-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-20T09:02:16,963 DEBUG [RS_CLOSE_META-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-20T09:02:16,963 DEBUG [RS_CLOSE_META-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-20T09:02:16,963 DEBUG [RS_CLOSE_META-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-20T09:02:16,963 INFO [RS_CLOSE_META-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=19.91 KB heapSize=35.20 KB 2024-11-20T09:02:16,984 DEBUG [RS_CLOSE_META-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/hbase/meta/1588230740/.tmp/info/d784760f78294d29a686039727644661 is 95, key is TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb./info:/1732093198192/DeleteFamily/seqid=0 2024-11-20T09:02:16,987 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742464_1640 (size=6514) 2024-11-20T09:02:17,055 INFO [regionserver/be1eb2620e0e:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-20T09:02:17,163 DEBUG [RS:0;be1eb2620e0e:34223 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-20T09:02:17,363 DEBUG [RS:0;be1eb2620e0e:34223 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-20T09:02:17,388 INFO [RS_CLOSE_META-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.75 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/hbase/meta/1588230740/.tmp/info/d784760f78294d29a686039727644661 2024-11-20T09:02:17,405 DEBUG [RS_CLOSE_META-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/hbase/meta/1588230740/.tmp/ns/0775440b286843d28e7a2263283e81d9 is 93, key is TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb./ns:/1732093198151/DeleteFamily/seqid=0 2024-11-20T09:02:17,409 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742465_1641 (size=5906) 2024-11-20T09:02:17,494 INFO [regionserver/be1eb2620e0e:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-20T09:02:17,494 INFO [regionserver/be1eb2620e0e:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-20T09:02:17,563 DEBUG [RS:0;be1eb2620e0e:34223 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-20T09:02:17,764 DEBUG [RS:0;be1eb2620e0e:34223 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-20T09:02:17,809 INFO [RS_CLOSE_META-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=608 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/hbase/meta/1588230740/.tmp/ns/0775440b286843d28e7a2263283e81d9 2024-11-20T09:02:17,827 DEBUG [RS_CLOSE_META-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/hbase/meta/1588230740/.tmp/rep_barrier/0a570c21c0074f3dae1e704a8e51654b is 102, key is TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb./rep_barrier:/1732093198151/DeleteFamily/seqid=0 2024-11-20T09:02:17,830 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742466_1642 (size=6025) 2024-11-20T09:02:17,964 DEBUG [RS:0;be1eb2620e0e:34223 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-20T09:02:18,164 DEBUG [RS:0;be1eb2620e0e:34223 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-20T09:02:18,193 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-20T09:02:18,231 INFO [RS_CLOSE_META-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=588 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/hbase/meta/1588230740/.tmp/rep_barrier/0a570c21c0074f3dae1e704a8e51654b 2024-11-20T09:02:18,248 DEBUG [RS_CLOSE_META-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/hbase/meta/1588230740/.tmp/table/60280dbcdfef437eafb77e4923f0717c is 96, key is TestAcidGuarantees,,1732093163518.1c52b94a9e6adce80d2d034c31b67cfb./table:/1732093198151/DeleteFamily/seqid=0 2024-11-20T09:02:18,251 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742467_1643 (size=5936) 2024-11-20T09:02:18,364 DEBUG [RS:0;be1eb2620e0e:34223 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-20T09:02:18,564 DEBUG [RS:0;be1eb2620e0e:34223 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-20T09:02:18,652 INFO [RS_CLOSE_META-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.99 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/hbase/meta/1588230740/.tmp/table/60280dbcdfef437eafb77e4923f0717c 2024-11-20T09:02:18,655 DEBUG [RS_CLOSE_META-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/hbase/meta/1588230740/.tmp/info/d784760f78294d29a686039727644661 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/hbase/meta/1588230740/info/d784760f78294d29a686039727644661 2024-11-20T09:02:18,658 INFO [RS_CLOSE_META-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/hbase/meta/1588230740/info/d784760f78294d29a686039727644661, entries=12, sequenceid=90, filesize=6.4 K 2024-11-20T09:02:18,658 DEBUG [RS_CLOSE_META-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/hbase/meta/1588230740/.tmp/ns/0775440b286843d28e7a2263283e81d9 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/hbase/meta/1588230740/ns/0775440b286843d28e7a2263283e81d9 2024-11-20T09:02:18,661 INFO [RS_CLOSE_META-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/hbase/meta/1588230740/ns/0775440b286843d28e7a2263283e81d9, entries=8, sequenceid=90, filesize=5.8 K 2024-11-20T09:02:18,661 DEBUG [RS_CLOSE_META-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/hbase/meta/1588230740/.tmp/rep_barrier/0a570c21c0074f3dae1e704a8e51654b as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/hbase/meta/1588230740/rep_barrier/0a570c21c0074f3dae1e704a8e51654b 2024-11-20T09:02:18,664 INFO [RS_CLOSE_META-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/hbase/meta/1588230740/rep_barrier/0a570c21c0074f3dae1e704a8e51654b, entries=6, sequenceid=90, filesize=5.9 K 2024-11-20T09:02:18,664 DEBUG [RS_CLOSE_META-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/hbase/meta/1588230740/.tmp/table/60280dbcdfef437eafb77e4923f0717c as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/hbase/meta/1588230740/table/60280dbcdfef437eafb77e4923f0717c 2024-11-20T09:02:18,666 INFO [RS_CLOSE_META-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/hbase/meta/1588230740/table/60280dbcdfef437eafb77e4923f0717c, entries=7, sequenceid=90, filesize=5.8 K 2024-11-20T09:02:18,667 INFO [RS_CLOSE_META-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1704ms, sequenceid=90, compaction requested=false 2024-11-20T09:02:18,671 DEBUG [RS_CLOSE_META-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/data/hbase/meta/1588230740/recovered.edits/93.seqid, newMaxSeqId=93, maxSeqId=1 2024-11-20T09:02:18,671 DEBUG [RS_CLOSE_META-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-20T09:02:18,671 INFO [RS_CLOSE_META-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-20T09:02:18,671 DEBUG [RS_CLOSE_META-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732093336962Running coprocessor pre-close hooks at 1732093336963 (+1 ms)Disabling compacts and flushes for region at 1732093336963Disabling writes for close at 1732093336963Obtaining lock to block concurrent updates at 1732093336963Preparing flush snapshotting stores in 1588230740 at 1732093336963Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=20384, getHeapSize=35984, getOffHeapSize=0, getCellsCount=191 at 1732093336963Flushing stores of hbase:meta,,1.1588230740 at 1732093336964 (+1 ms)Flushing 1588230740/info: creating writer at 1732093336964Flushing 1588230740/info: appending metadata at 1732093336984 (+20 ms)Flushing 1588230740/info: closing flushed file at 1732093336984Flushing 1588230740/ns: creating writer at 1732093337391 (+407 ms)Flushing 1588230740/ns: appending metadata at 1732093337405 (+14 ms)Flushing 1588230740/ns: closing flushed file at 1732093337405Flushing 1588230740/rep_barrier: creating writer at 1732093337812 (+407 ms)Flushing 1588230740/rep_barrier: appending metadata at 1732093337827 (+15 ms)Flushing 1588230740/rep_barrier: closing flushed file at 1732093337827Flushing 1588230740/table: creating writer at 1732093338234 (+407 ms)Flushing 1588230740/table: appending metadata at 1732093338248 (+14 ms)Flushing 1588230740/table: closing flushed file at 1732093338248Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@46a65ace: reopening flushed file at 1732093338655 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4f85ad83: reopening flushed file at 1732093338658 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2c1209e: reopening flushed file at 1732093338661 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1b705659: reopening flushed file at 1732093338664 (+3 ms)Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1704ms, sequenceid=90, compaction requested=false at 1732093338667 (+3 ms)Writing region close event to WAL at 1732093338668 (+1 ms)Running coprocessor post-close hooks at 1732093338671 (+3 ms)Closed at 1732093338671 2024-11-20T09:02:18,671 DEBUG [RS_CLOSE_META-regionserver/be1eb2620e0e:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-20T09:02:18,765 INFO [RS:0;be1eb2620e0e:34223 {}] regionserver.HRegionServer(976): stopping server be1eb2620e0e,34223,1732093159960; all regions closed. 2024-11-20T09:02:18,769 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073741834_1010 (size=25344) 2024-11-20T09:02:18,771 DEBUG [RS:0;be1eb2620e0e:34223 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/oldWALs 2024-11-20T09:02:18,771 INFO [RS:0;be1eb2620e0e:34223 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL be1eb2620e0e%2C34223%2C1732093159960.meta:.meta(num 1732093163074) 2024-11-20T09:02:18,773 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073741833_1009 (size=14641544) 2024-11-20T09:02:18,775 DEBUG [RS:0;be1eb2620e0e:34223 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/oldWALs 2024-11-20T09:02:18,775 INFO [RS:0;be1eb2620e0e:34223 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL be1eb2620e0e%2C34223%2C1732093159960:(num 1732093162659) 2024-11-20T09:02:18,775 DEBUG [RS:0;be1eb2620e0e:34223 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T09:02:18,775 INFO [RS:0;be1eb2620e0e:34223 {}] regionserver.LeaseManager(133): Closed leases 2024-11-20T09:02:18,775 INFO [RS:0;be1eb2620e0e:34223 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-20T09:02:18,775 INFO [RS:0;be1eb2620e0e:34223 {}] hbase.ChoreService(370): Chore service for: regionserver/be1eb2620e0e:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-20T09:02:18,776 INFO [RS:0;be1eb2620e0e:34223 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-20T09:02:18,776 INFO [regionserver/be1eb2620e0e:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-20T09:02:18,776 INFO [RS:0;be1eb2620e0e:34223 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:34223 2024-11-20T09:02:18,780 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34223-0x1001427dc570001, quorum=127.0.0.1:52155, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/be1eb2620e0e,34223,1732093159960 2024-11-20T09:02:18,780 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44373-0x1001427dc570000, quorum=127.0.0.1:52155, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-20T09:02:18,780 INFO [RS:0;be1eb2620e0e:34223 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-20T09:02:18,783 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [be1eb2620e0e,34223,1732093159960] 2024-11-20T09:02:18,785 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/be1eb2620e0e,34223,1732093159960 already deleted, retry=false 2024-11-20T09:02:18,785 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; be1eb2620e0e,34223,1732093159960 expired; onlineServers=0 2024-11-20T09:02:18,786 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'be1eb2620e0e,44373,1732093159150' ***** 2024-11-20T09:02:18,786 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-20T09:02:18,786 INFO [M:0;be1eb2620e0e:44373 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-20T09:02:18,786 INFO [M:0;be1eb2620e0e:44373 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-20T09:02:18,786 DEBUG [M:0;be1eb2620e0e:44373 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-20T09:02:18,786 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-20T09:02:18,786 DEBUG [M:0;be1eb2620e0e:44373 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-20T09:02:18,786 DEBUG [master/be1eb2620e0e:0:becomeActiveMaster-HFileCleaner.small.0-1732093162340 {}] cleaner.HFileCleaner(306): Exit Thread[master/be1eb2620e0e:0:becomeActiveMaster-HFileCleaner.small.0-1732093162340,5,FailOnTimeoutGroup] 2024-11-20T09:02:18,786 DEBUG [master/be1eb2620e0e:0:becomeActiveMaster-HFileCleaner.large.0-1732093162338 {}] cleaner.HFileCleaner(306): Exit Thread[master/be1eb2620e0e:0:becomeActiveMaster-HFileCleaner.large.0-1732093162338,5,FailOnTimeoutGroup] 2024-11-20T09:02:18,786 INFO [M:0;be1eb2620e0e:44373 {}] hbase.ChoreService(370): Chore service for: master/be1eb2620e0e:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-20T09:02:18,786 INFO [M:0;be1eb2620e0e:44373 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-20T09:02:18,786 DEBUG [M:0;be1eb2620e0e:44373 {}] master.HMaster(1795): Stopping service threads 2024-11-20T09:02:18,786 INFO [M:0;be1eb2620e0e:44373 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-20T09:02:18,787 INFO [M:0;be1eb2620e0e:44373 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T09:02:18,787 ERROR [M:0;be1eb2620e0e:44373 {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[IPC Client (1102635514) connection to localhost/127.0.0.1:40923 from jenkins,5,PEWorkerGroup] Thread[IPC Parameter Sending Thread for localhost/127.0.0.1:40923,5,PEWorkerGroup] Thread[HFileArchiver-6,5,PEWorkerGroup] 2024-11-20T09:02:18,787 INFO [M:0;be1eb2620e0e:44373 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-20T09:02:18,787 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-20T09:02:18,788 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44373-0x1001427dc570000, quorum=127.0.0.1:52155, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-20T09:02:18,788 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44373-0x1001427dc570000, quorum=127.0.0.1:52155, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T09:02:18,788 DEBUG [M:0;be1eb2620e0e:44373 {}] zookeeper.ZKUtil(347): master:44373-0x1001427dc570000, quorum=127.0.0.1:52155, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-20T09:02:18,788 WARN [M:0;be1eb2620e0e:44373 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-20T09:02:18,789 INFO [M:0;be1eb2620e0e:44373 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/.lastflushedseqids 2024-11-20T09:02:18,798 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742468_1644 (size=72) 2024-11-20T09:02:18,884 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34223-0x1001427dc570001, quorum=127.0.0.1:52155, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-20T09:02:18,884 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34223-0x1001427dc570001, quorum=127.0.0.1:52155, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-20T09:02:18,884 INFO [RS:0;be1eb2620e0e:34223 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-20T09:02:18,884 INFO [RS:0;be1eb2620e0e:34223 {}] regionserver.HRegionServer(1031): Exiting; stopping=be1eb2620e0e,34223,1732093159960; zookeeper connection closed. 2024-11-20T09:02:18,885 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@316697ae {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@316697ae 2024-11-20T09:02:18,885 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-20T09:02:19,198 INFO [M:0;be1eb2620e0e:44373 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-20T09:02:19,199 INFO [M:0;be1eb2620e0e:44373 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-20T09:02:19,199 DEBUG [M:0;be1eb2620e0e:44373 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-20T09:02:19,199 INFO [M:0;be1eb2620e0e:44373 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T09:02:19,199 DEBUG [M:0;be1eb2620e0e:44373 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T09:02:19,199 DEBUG [M:0;be1eb2620e0e:44373 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-20T09:02:19,199 DEBUG [M:0;be1eb2620e0e:44373 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T09:02:19,199 INFO [M:0;be1eb2620e0e:44373 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=763.03 KB heapSize=938.66 KB 2024-11-20T09:02:19,214 DEBUG [M:0;be1eb2620e0e:44373 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/f9a8c350d6d74215888757ac8338ea2f is 82, key is hbase:meta,,1/info:regioninfo/1732093163171/Put/seqid=0 2024-11-20T09:02:19,217 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742469_1645 (size=5672) 2024-11-20T09:02:19,618 INFO [M:0;be1eb2620e0e:44373 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=2177 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/f9a8c350d6d74215888757ac8338ea2f 2024-11-20T09:02:19,637 DEBUG [M:0;be1eb2620e0e:44373 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/e3c073608bcf4a6e965696996f71ca52 is 2276, key is \x00\x00\x00\x00\x00\x00\x00\x8B/proc:d/1732093311881/Put/seqid=0 2024-11-20T09:02:19,641 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742470_1646 (size=42736) 2024-11-20T09:02:19,645 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-20T09:02:19,645 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-20T09:02:19,646 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-20T09:02:20,041 INFO [M:0;be1eb2620e0e:44373 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=762.47 KB at sequenceid=2177 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/e3c073608bcf4a6e965696996f71ca52 2024-11-20T09:02:20,045 INFO [M:0;be1eb2620e0e:44373 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for e3c073608bcf4a6e965696996f71ca52 2024-11-20T09:02:20,059 DEBUG [M:0;be1eb2620e0e:44373 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/ea91d9b4e02c47a1abd9c6ec2cb151b3 is 69, key is be1eb2620e0e,34223,1732093159960/rs:state/1732093162414/Put/seqid=0 2024-11-20T09:02:20,062 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073742471_1647 (size=5156) 2024-11-20T09:02:20,463 INFO [M:0;be1eb2620e0e:44373 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=2177 (bloomFilter=true), to=hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/ea91d9b4e02c47a1abd9c6ec2cb151b3 2024-11-20T09:02:20,466 DEBUG [M:0;be1eb2620e0e:44373 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/f9a8c350d6d74215888757ac8338ea2f as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/f9a8c350d6d74215888757ac8338ea2f 2024-11-20T09:02:20,468 INFO [M:0;be1eb2620e0e:44373 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/f9a8c350d6d74215888757ac8338ea2f, entries=8, sequenceid=2177, filesize=5.5 K 2024-11-20T09:02:20,469 DEBUG [M:0;be1eb2620e0e:44373 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/e3c073608bcf4a6e965696996f71ca52 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/e3c073608bcf4a6e965696996f71ca52 2024-11-20T09:02:20,471 INFO [M:0;be1eb2620e0e:44373 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for e3c073608bcf4a6e965696996f71ca52 2024-11-20T09:02:20,471 INFO [M:0;be1eb2620e0e:44373 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/e3c073608bcf4a6e965696996f71ca52, entries=162, sequenceid=2177, filesize=41.7 K 2024-11-20T09:02:20,472 DEBUG [M:0;be1eb2620e0e:44373 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/ea91d9b4e02c47a1abd9c6ec2cb151b3 as hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/ea91d9b4e02c47a1abd9c6ec2cb151b3 2024-11-20T09:02:20,474 INFO [M:0;be1eb2620e0e:44373 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40923/user/jenkins/test-data/7a1e95ad-a9a4-6101-508f-8aee1c104a0a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/ea91d9b4e02c47a1abd9c6ec2cb151b3, entries=1, sequenceid=2177, filesize=5.0 K 2024-11-20T09:02:20,475 INFO [M:0;be1eb2620e0e:44373 {}] regionserver.HRegion(3140): Finished flush of dataSize ~763.03 KB/781338, heapSize ~938.37 KB/960888, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1276ms, sequenceid=2177, compaction requested=false 2024-11-20T09:02:20,476 INFO [M:0;be1eb2620e0e:44373 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T09:02:20,476 DEBUG [M:0;be1eb2620e0e:44373 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732093339199Disabling compacts and flushes for region at 1732093339199Disabling writes for close at 1732093339199Obtaining lock to block concurrent updates at 1732093339199Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732093339199Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=781338, getHeapSize=961128, getOffHeapSize=0, getCellsCount=2364 at 1732093339199Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732093339200 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732093339200Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732093339214 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732093339214Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732093339621 (+407 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732093339637 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732093339637Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732093340045 (+408 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732093340059 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732093340059Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2a04f627: reopening flushed file at 1732093340465 (+406 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2897ebba: reopening flushed file at 1732093340469 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7a200b64: reopening flushed file at 1732093340471 (+2 ms)Finished flush of dataSize ~763.03 KB/781338, heapSize ~938.37 KB/960888, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1276ms, sequenceid=2177, compaction requested=false at 1732093340475 (+4 ms)Writing region close event to WAL at 1732093340476 (+1 ms)Closed at 1732093340476 2024-11-20T09:02:20,478 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40033 is added to blk_1073741830_1006 (size=924010) 2024-11-20T09:02:20,479 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-20T09:02:20,479 INFO [M:0;be1eb2620e0e:44373 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-20T09:02:20,479 INFO [M:0;be1eb2620e0e:44373 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:44373 2024-11-20T09:02:20,479 INFO [M:0;be1eb2620e0e:44373 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-20T09:02:20,581 INFO [M:0;be1eb2620e0e:44373 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-20T09:02:20,581 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44373-0x1001427dc570000, quorum=127.0.0.1:52155, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-20T09:02:20,582 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44373-0x1001427dc570000, quorum=127.0.0.1:52155, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-20T09:02:20,587 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4269bd31{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-20T09:02:20,589 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@353d64f2{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-20T09:02:20,589 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-20T09:02:20,589 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@435f9f96{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-20T09:02:20,589 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4fd7fa7c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d52d975-8094-4c70-a6d2-7d2f656df145/hadoop.log.dir/,STOPPED} 2024-11-20T09:02:20,592 WARN [BP-1443621306-172.17.0.2-1732093156059 heartbeating to localhost/127.0.0.1:40923 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-20T09:02:20,592 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-20T09:02:20,592 WARN [BP-1443621306-172.17.0.2-1732093156059 heartbeating to localhost/127.0.0.1:40923 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1443621306-172.17.0.2-1732093156059 (Datanode Uuid 4ae02408-43a8-4abb-8a76-2d8c0890ff10) service to localhost/127.0.0.1:40923 2024-11-20T09:02:20,592 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-20T09:02:20,595 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d52d975-8094-4c70-a6d2-7d2f656df145/cluster_0d88e112-ef41-ee03-4061-1e732edd46d6/data/data1/current/BP-1443621306-172.17.0.2-1732093156059 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-20T09:02:20,595 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d52d975-8094-4c70-a6d2-7d2f656df145/cluster_0d88e112-ef41-ee03-4061-1e732edd46d6/data/data2/current/BP-1443621306-172.17.0.2-1732093156059 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-20T09:02:20,596 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-20T09:02:20,604 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4ded99d3{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-20T09:02:20,604 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-20T09:02:20,604 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-20T09:02:20,605 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-20T09:02:20,605 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2d52d975-8094-4c70-a6d2-7d2f656df145/hadoop.log.dir/,STOPPED} 2024-11-20T09:02:20,621 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-20T09:02:20,740 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down